@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1

package/dist/src/report/experiments/renderAgentSuccessRateHtml.js

@@ -0,0 +1,171 @@
+import { AGENT_SUCCESS_BASIS_LABEL, formatAgentSuccessMetric, formatAgentSuccessPairs, formatAgentSuccessTotal, renderCampaignEvidenceSummary } from "./renderAgentSuccessRateText.js";
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+/**
+ * HTML rendering of the typed section. All dynamic content is escaped, artifact references are plain text (never links
+ * or embedded files), and no value is recalculated here.
+ */
+export function renderAgentSuccessRateHtml(report) {
+    if (!report)
+        return "";
+    const realMode = report.identity.executionMode === "real-agent";
+    const m = formatAgentSuccessMetric;
+    const parts = [];
+    if (report.campaignEvidence)
+        parts.push(`<section><h2>Campaign evidence</h2><pre>${escapeHtml(renderCampaignEvidenceSummary(report))}</pre></section>`);
+    parts.push(`<section id="agent-success-rate">
+    <h2>Agent Success Rate</h2>
+    <h3>Experiment identity</h3>
+    ${table(["Field", "Value"], [
+        ["Run ID", report.identity.runId],
+        ["Plugin", `${report.identity.pluginId} (schema ${report.identity.pluginSchemaVersion})`],
+        ["Run status", report.identity.runStatus],
+        ["Cases", String(report.identity.caseCount)],
+        ["Treatments", report.identity.treatmentOrder.join(", ")],
+        ["Treatment outcomes", String(report.identity.treatmentOutcomeCount)]
+    ])}
+    <h3>Execution mode and provider</h3>
+    ${table(["Field", "Value"], [
+        ["Execution mode", report.identity.executionMode],
+        ["Provider", report.identity.providerId ?? "none (no coding agent was invoked)"],
+        ["Provider timeout per attempt", report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`],
+        ["Repair attempts allowed", report.identity.repairAttempts === null ? "not applicable" : String(report.identity.repairAttempts)],
+        ["Maximum attempts per treatment", String(report.identity.maxAttemptsPerTreatment)]
+    ])}
+    <h3>Scientific limitations</h3>
+    <p><strong>${escapeHtml(report.scientificStatement)}</strong></p>
+    <ul>${report.limitations.map((limitation) => `<li>${escapeHtml(limitation)}</li>`).join("")}</ul>
+    <h3>Overall result availability</h3>
+    ${table(["Field", "Value"], [
+        ["Overall", report.resultAvailability.overall],
+        ["Outcomes with a determinate verdict", `${report.resultAvailability.evaluableTreatmentOutcomes} of ${report.resultAvailability.totalTreatmentOutcomes}`],
+        ["Outcomes without a determinate verdict", String(report.resultAvailability.unavailableTreatmentOutcomes)]
+    ])}
+    ${report.resultAvailability.notes.length > 0 ? `<ul>${report.resultAvailability.notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")}</ul>` : ""}
+  </section>`);
+    const treatmentRows = report.treatments.map((treatment) => [
+        treatment.treatmentId,
+        String(treatment.evaluableCases),
+        String(treatment.initialSuccessfulCases),
+        m(treatment.initialSuccessRate),
+        String(treatment.finalSuccessfulCases),
+        m(treatment.finalSuccessRate),
+        treatment.repair ? String(treatment.repair.repairAttempts) : "not applicable",
+        treatment.repair ? String(treatment.repair.repairedCases) : "not applicable"
+    ]);
+    const comparison = report.comparison;
+    parts.push(`<section id="agent-success-rate-comparison">
+    <h2>Treatment Comparison</h2>
+    ${table(["Treatment", "Evaluable cases", "Initial successful cases", "Initial success rate", "Final successful cases", "Final success rate", "Repair attempts", "Repaired cases"], treatmentRows)}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .map((treatment) => `<p class="muted">${escapeHtml(treatment.treatmentId)}: repair-eligible cases ${treatment.repair.repairEligibleCases}; cases with a repair ${treatment.repair.repairAttemptedCases}; repair success rate ${escapeHtml(m(treatment.repair.repairSuccessRate))}; mean attempts per evaluable case ${escapeHtml(m(treatment.repair.meanAttemptsPerEvaluableCase))}.</p>`)
+        .join("")}
+    ${comparison
+        ? `<h3>Matched comparison (descriptive; no winner is declared)</h3>
+    ${table(["Statement", "Value"], [
+            ["Comparison evaluable", comparison.comparisonEvaluable ? "yes" : "no"],
+            ["Observed outcome difference between treatments", comparison.observedOutcomeDifference],
+            ["Statistical or causal effect", `${comparison.statisticalEffect} (no significance or causal claim is made)`],
+            ["Final matched cases", comparison.finalMatchedCaseIds.join(", ") || "none"],
+            ["Final paired outcomes", formatAgentSuccessPairs(comparison.finalPairedOutcomes)],
+            ["Initial-attempt matched cases", comparison.initialMatchedCaseIds ? comparison.initialMatchedCaseIds.join(", ") || "none" : "not applicable"],
+            ["Initial-attempt paired outcomes", formatAgentSuccessPairs(comparison.initialPairedOutcomes)]
+        ])}
+    ${comparison.pairedDifferences.length > 0
+            ? table(["Measurement", "Basis", "Matched cases", "raw-full-file mean", "context-pack mean", "context-pack minus raw-full-file"], comparison.pairedDifferences.map((difference) => [
+                difference.id,
+                AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis,
+                String(difference.matchedCaseIds.length),
+                m(difference.rawFullFileMean),
+                m(difference.contextPackMean),
+                m(difference.meanDifference)
+            ]))
+            : ""}`
+        : `<p class="muted">Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).</p>`}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cases">
+    <h2>Per-Case Results</h2>
+    ${table(["Case", "Project", "Locality", "Treatment", "Status", "Initial task success", "Final task success", "Attempts", "Task-check pass rate", "Regression-check pass rate", "Required-fact coverage"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality,
+        treatment.treatmentId,
+        `${treatment.executionStatus} / ${treatment.evidenceAvailability}`,
+        m(treatment.initialTaskSuccess),
+        m(treatment.finalTaskSuccess),
+        String(treatment.attemptCount),
+        m(treatment.taskCheckPassRate),
+        m(treatment.regressionCheckPassRate),
+        m(treatment.requiredFactCoverage)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-repair-history">
+    <h2>Per-Attempt Repair History</h2>
+    ${!realMode || report.repairHistory.length === 0
+        ? `<p class="muted">${realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts."}</p>`
+        : table(["Case", "Treatment", "Attempt", "Provider status", "Patch outcome", "Verification", "Task success", "Outcome", "Duration", "Tokens", "Artifacts"], report.repairHistory.flatMap((history) => history.attempts.map((attempt) => [
+            history.caseId,
+            history.treatmentId,
+            String(attempt.attemptNumber),
+            attempt.providerStatus,
+            attempt.patchOutcome,
+            attempt.verificationAvailability,
+            m(attempt.taskSuccess),
+            `${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`,
+            m(attempt.providerDurationMs),
+            attempt.tokenAvailability === "available" ? m(attempt.providerTokens) : "unavailable",
+            `proposed: ${attempt.proposedPatchPath ?? "none"}; applied: ${attempt.appliedPatchPath ?? "none"}; agent: ${attempt.agentArtifactDirectory ?? "none"}`
+        ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-edit-quality">
+    <h2>Edit Quality And Blast Radius (final evaluated patch)</h2>
+    ${table(["Case", "Treatment", "Expected-edit coverage", "Edit-scope precision", "Unexpected changed files", "Protected mutations", "Changed files", "Total churn", "Relative churn"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.expectedEditCoverage),
+        m(treatment.editScopePrecision),
+        m(treatment.unexpectedChangedFiles),
+        m(treatment.protectedMutations),
+        `${m(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}`,
+        m(treatment.totalChurn),
+        m(treatment.relativeChurn)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cost">
+    <h2>Duration And Token Evidence</h2>
+    ${table(["Case", "Treatment", "Final-attempt provider duration", "Final-attempt provider tokens", "Total provider duration (all attempts)", "Total provider tokens (all attempts)"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.finalAttemptProviderDurationMs),
+        m(treatment.finalAttemptProviderTokens),
+        treatment.totalProviderDurationMs ? m(treatment.totalProviderDurationMs) : "not applicable",
+        treatment.totalProviderTokens ? m(treatment.totalProviderTokens) : "not applicable"
+    ])))}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .flatMap((treatment) => [
+        ...treatment.repair.providerDurationMs.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider duration [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`),
+        ...treatment.repair.providerTokens.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider tokens [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`)
+    ])
+        .join("")}
+  </section>`);
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => [entry.caseId, treatment.treatmentId, String(error.attemptNumber), error.code, error.message])));
+    parts.push(`<section id="agent-success-rate-findings">
+    <h2>Agent Success Rate Warnings, Failures And Artifacts</h2>
+    ${report.warnings.length === 0 && errors.length === 0 ? "<p>No warnings or recorded errors.</p>" : table(["Source", "Case", "Treatment", "Attempt", "Code", "Message"], [...report.warnings.map((warning) => ["warning", "", "", "", warning.code, warning.message]), ...errors.map((error) => ["error", ...error])])}
+    ${table(["Artifact", "Reference", "Case", "Treatment"], report.artifacts.map((artifact) => [artifact.id, artifact.path ?? "[reference withheld]", artifact.caseId ?? "", artifact.variantId ?? ""]))}
+  </section>`);
+    return parts.join("\n\n  ");
+}

package/dist/src/report/experiments/renderAgentSuccessRateText.js

@@ -0,0 +1,146 @@
+export function renderCampaignEvidenceSummary(report) {
+    const m = report.campaignEvidence;
+    return m ? [`Campaign evidence: ${m.state}; checkpoint revision ${m.checkpointGeneration} (${m.checkpointRevision}); completed evaluations ${m.unitCounts.completed}/${m.totalTreatmentUnits}; completed cases ${m.caseCounts.completed}/${m.selectedCaseCount}`,
+        ...m.comparisons.map(q => `${q.quantity}: ${q.quality}; ${q.basis}; ${q.measurementBasis}; matched ${q.matchedCaseIds.length}/${q.selectedCaseCount}; excluded ${q.excludedCaseIds.length}; raw=${q.rawFullFileMean ?? "unavailable"}; context=${q.contextPackMean ?? "unavailable"}; delta=${q.meanDifference ?? "unavailable"}; unit=${q.unit}; reasons=${q.reasons.join(", ") || "none"}; ${q.reasonExplanations.join(" ")}`)].join("\n") : "";
+}
+/** One display form for every metric: an absent value is labelled, never rendered as zero. */
+export function formatAgentSuccessMetric(metric) {
+    if (!metric)
+        return "unavailable (not recorded)";
+    if (metric.availability === "unavailable")
+        return `unavailable (${metric.reason ?? "no reason recorded"})`;
+    if (metric.availability === "not-applicable")
+        return `not applicable (${metric.reason ?? "no reason recorded"})`;
+    const value = metric.value;
+    if (typeof value === "boolean")
+        return value ? "yes" : "no";
+    if (typeof value !== "number")
+        return "unavailable (no value)";
+    switch (metric.unit) {
+        case "ratio":
+            return value.toFixed(4);
+        case "ms":
+            return `${Math.round(value)} ms`;
+        case "tokens":
+            return `${Math.round(value)} tokens`;
+        case "lines":
+            return `${Math.round(value)} lines`;
+        default:
+            return Number.isInteger(value) ? String(value) : value.toFixed(4);
+    }
+}
+export function formatAgentSuccessTotal(total) {
+    const base = `${formatAgentSuccessMetric(total.total)}; contributing cases=${total.contributingCaseCount}, measured=${total.availableCaseCount}, unmeasured=${total.unavailableCaseCount}`;
+    return total.unavailableCaseCount > 0 && total.sumOfAvailable !== null ? `${base}; sum of measured cases only=${total.sumOfAvailable} (not a complete total)` : base;
+}
+export function formatAgentSuccessPairs(pairs) {
+    if (!pairs)
+        return "not recorded";
+    return `both succeeded=${pairs.bothSucceeded}, only raw-full-file=${pairs.onlyRawFullFileSucceeded}, only context-pack=${pairs.onlyContextPackSucceeded}, neither=${pairs.neitherSucceeded}`;
+}
+export const AGENT_SUCCESS_BASIS_LABEL = {
+    "first-attempt": "first attempt",
+    "final-attempt": "final attempt",
+    "total-across-attempts": "total across attempts"
+};
+const sanitize = (value) => String(value ?? "").replace(/[\r\n\t]+/g, " ").trim();
+/** Text rendering of the typed section. It formats already-calculated values and recalculates none of them. */
+export function renderAgentSuccessRateTextLines(report) {
+    const realMode = report.identity.executionMode === "real-agent";
+    const lines = [];
+    const section = (title) => {
+        lines.push("", title, "-".repeat(title.length));
+    };
+    lines.push("", "AGENT SUCCESS RATE", "==================");
+    section("A. Experiment identity");
+    lines.push(`Run ID: ${sanitize(report.identity.runId)}`, `Plugin: ${sanitize(report.identity.pluginId)} (schema ${sanitize(report.identity.pluginSchemaVersion)})`, `Run status: ${sanitize(report.identity.runStatus)}`, `Cases: ${report.identity.caseCount}; treatments: ${report.identity.treatmentOrder.join(", ")}; treatment outcomes: ${report.identity.treatmentOutcomeCount}`);
+    section("B. Execution mode and provider");
+    lines.push(`Execution mode: ${sanitize(report.identity.executionMode)}`, `Provider: ${report.identity.providerId ?? "none (no coding agent was invoked)"}`, `Provider timeout per attempt: ${report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`}`, `Repair attempts allowed: ${report.identity.repairAttempts === null ? "not applicable" : report.identity.repairAttempts}; maximum attempts per treatment: ${report.identity.maxAttemptsPerTreatment}`);
+    section("C. Scientific limitations");
+    lines.push(sanitize(report.scientificStatement));
+    for (const limitation of report.limitations)
+        lines.push(`- ${sanitize(limitation)}`);
+    section("D. Overall result availability");
+    const availability = report.resultAvailability;
+    lines.push(`Overall: ${availability.overall}`, `Treatment outcomes with a determinate task-success verdict: ${availability.evaluableTreatmentOutcomes} of ${availability.totalTreatmentOutcomes}; without: ${availability.unavailableTreatmentOutcomes}`);
+    for (const note of availability.notes)
+        lines.push(`- ${sanitize(note)}`);
+    section("E. Treatment comparison");
+    for (const treatment of report.treatments) {
+        lines.push(`Treatment ${treatment.treatmentId}:`, `  evaluable cases: ${treatment.evaluableCases}`, `  initial-attempt successful cases: ${treatment.initialSuccessfulCases}; initial success rate: ${formatAgentSuccessMetric(treatment.initialSuccessRate)}`, `  final successful cases: ${treatment.finalSuccessfulCases}; final success rate: ${formatAgentSuccessMetric(treatment.finalSuccessRate)}`);
+        if (treatment.repair) {
+            lines.push(`  repair-eligible cases: ${treatment.repair.repairEligibleCases}; repair attempts executed: ${treatment.repair.repairAttempts}; cases with a repair: ${treatment.repair.repairAttemptedCases}; repaired cases: ${treatment.repair.repairedCases}`, `  repair success rate: ${formatAgentSuccessMetric(treatment.repair.repairSuccessRate)}; mean attempts per evaluable case: ${formatAgentSuccessMetric(treatment.repair.meanAttemptsPerEvaluableCase)}`);
+        }
+        else {
+            lines.push("  repair: not applicable (deterministic-fixture mode; the single fixture attempt is both initial and final)");
+        }
+    }
+    if (report.comparison) {
+        const comparison = report.comparison;
+        lines.push("Matched comparison (descriptive; no winner is declared):", `  comparison evaluable: ${comparison.comparisonEvaluable ? "yes" : "no"}`, `  observed outcome difference between treatments: ${comparison.observedOutcomeDifference}`, `  statistical or causal effect: ${comparison.statisticalEffect} (no significance or causal claim is made)`, `  final matched cases: ${comparison.finalMatchedCaseIds.length > 0 ? comparison.finalMatchedCaseIds.join(", ") : "none"}`, `  final paired outcomes: ${formatAgentSuccessPairs(comparison.finalPairedOutcomes)}`);
+        if (comparison.initialMatchedCaseIds) {
+            lines.push(`  initial-attempt matched cases: ${comparison.initialMatchedCaseIds.length > 0 ? comparison.initialMatchedCaseIds.join(", ") : "none"}`, `  initial-attempt paired outcomes: ${formatAgentSuccessPairs(comparison.initialPairedOutcomes)}`);
+        }
+        for (const incomplete of comparison.finalIncompleteCases)
+            lines.push(`  excluded from final matched metrics: ${incomplete.caseId} (no determinate verdict for ${incomplete.unavailableTreatmentIds.join(", ")})`);
+        for (const difference of comparison.pairedDifferences) {
+            lines.push(`  paired difference ${difference.id} [${AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis}] over ${difference.matchedCaseIds.length} matched case(s): raw-full-file ${formatAgentSuccessMetric(difference.rawFullFileMean)}; context-pack ${formatAgentSuccessMetric(difference.contextPackMean)}; context-pack minus raw-full-file ${formatAgentSuccessMetric(difference.meanDifference)}`);
+        }
+    }
+    else {
+        lines.push("Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).");
+    }
+    section("F. Per-case results");
+    for (const entry of report.cases) {
+        lines.push(`Case ${entry.caseId} (${sanitize(entry.caseName)}) project=${entry.benchmarkProject} locality=${entry.taskLocality}`);
+        for (const treatment of entry.treatments) {
+            lines.push(`  ${treatment.treatmentId}: status=${treatment.executionStatus} evidence=${treatment.evidenceAvailability}`, `    initial task success: ${formatAgentSuccessMetric(treatment.initialTaskSuccess)}; final task success: ${formatAgentSuccessMetric(treatment.finalTaskSuccess)}; attempts=${treatment.attemptCount}; repair attempts=${treatment.repairAttemptCount}; repair succeeded: ${formatAgentSuccessMetric(treatment.repairSucceeded)}`, `    task-check pass rate: ${formatAgentSuccessMetric(treatment.taskCheckPassRate)}; regression-check pass rate: ${formatAgentSuccessMetric(treatment.regressionCheckPassRate)}; required-fact coverage: ${formatAgentSuccessMetric(treatment.requiredFactCoverage)}`);
+        }
+    }
+    section("G. Per-attempt repair history");
+    if (!realMode || report.repairHistory.length === 0) {
+        lines.push(realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts.");
+    }
+    for (const history of report.repairHistory) {
+        lines.push(`Case ${history.caseId} / ${history.treatmentId}: ${history.attempts.length} attempt(s)`);
+        for (const attempt of history.attempts) {
+            lines.push(`  attempt ${attempt.attemptNumber}: provider=${attempt.providerStatus}; patch=${attempt.patchOutcome}; verification=${attempt.verificationAvailability}; task success=${formatAgentSuccessMetric(attempt.taskSuccess)}; outcome=${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`, `    duration: ${formatAgentSuccessMetric(attempt.providerDurationMs)}; tokens: ${attempt.tokenAvailability === "available" ? formatAgentSuccessMetric(attempt.providerTokens) : "unavailable"}`, `    artifacts: proposed patch=${attempt.proposedPatchPath ?? "none"}; applied patch=${attempt.appliedPatchPath ?? "none"}; agent files=${attempt.agentArtifactDirectory ?? "none"}`);
+        }
+    }
+    section("H. Edit-quality and blast-radius measurements (final evaluated patch)");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: expected-edit coverage=${formatAgentSuccessMetric(treatment.expectedEditCoverage)}; edit-scope precision=${formatAgentSuccessMetric(treatment.editScopePrecision)}; unexpected changed files=${formatAgentSuccessMetric(treatment.unexpectedChangedFiles)}; protected mutations=${formatAgentSuccessMetric(treatment.protectedMutations)}`, `  changed files=${formatAgentSuccessMetric(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}; total churn=${formatAgentSuccessMetric(treatment.totalChurn)}; relative churn=${formatAgentSuccessMetric(treatment.relativeChurn)}`);
+        }
+    }
+    section("I. Duration and token evidence");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: final-attempt provider duration=${formatAgentSuccessMetric(treatment.finalAttemptProviderDurationMs)}; final-attempt provider tokens=${formatAgentSuccessMetric(treatment.finalAttemptProviderTokens)}` +
+                (treatment.totalProviderDurationMs
+                    ? `; total provider duration across attempts=${formatAgentSuccessMetric(treatment.totalProviderDurationMs)}; total provider tokens across attempts=${formatAgentSuccessMetric(treatment.totalProviderTokens)}`
+                    : ""));
+        }
+    }
+    for (const treatment of report.treatments) {
+        if (!treatment.repair)
+            continue;
+        for (const total of treatment.repair.providerDurationMs)
+            lines.push(`${treatment.treatmentId} provider duration [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+        for (const total of treatment.repair.providerTokens)
+            lines.push(`${treatment.treatmentId} provider tokens [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+    }
+    section("J. Warnings, failures and artifact references");
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => ({ caseId: entry.caseId, treatmentId: treatment.treatmentId, ...error }))));
+    if (report.warnings.length === 0 && errors.length === 0)
+        lines.push("No warnings or recorded errors.");
+    for (const warning of report.warnings)
+        lines.push(`warning ${sanitize(warning.code)}: ${sanitize(warning.message)}`);
+    for (const error of errors)
+        lines.push(`error ${error.caseId} / ${error.treatmentId} attempt ${error.attemptNumber} ${sanitize(error.code)}: ${sanitize(error.message)}`);
+    for (const artifact of report.artifacts)
+        lines.push(`artifact ${sanitize(artifact.id)}: ${artifact.path ?? "[reference withheld]"}`);
+    if (report.campaignEvidence)
+        lines.push(renderCampaignEvidenceSummary(report));
+    return lines;
+}

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -5,6 +5,7 @@ import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml
 import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 import { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";
 import { renderContextPackGenerationHtml } from "./renderContextPackGenerationHtml.js";
+import { renderAgentSuccessRateHtml } from "./renderAgentSuccessRateHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
@@ -115,6 +116,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextPackGenerationHtml(report.contextPackGeneration ?? null)}
 
+  ${renderAgentSuccessRateHtml(report.agentSuccessRate ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -6,6 +6,7 @@ import { renderContextWindowScalingTextLines } from "./renderContextWindowScalin
 import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
 import { renderRetrievalQueryStrategyComparisonTextLines } from "./renderRetrievalQueryStrategyComparisonText.js";
 import { renderContextPackGenerationTextLines } from "./renderContextPackGenerationText.js";
+import { renderAgentSuccessRateTextLines } from "./renderAgentSuccessRateText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -691,6 +692,10 @@ export function renderPluginExperimentReportText(report) {
         lines.push(...renderRetrievalQueryStrategyComparisonTextLines(retrievalQueryStrategyComparison));
     }
     // Additive: other plugins keep their existing text output unchanged.
+    const agentSuccessRate = report.agentSuccessRate ?? null;
+    if (agentSuccessRate !== null) {
+        lines.push(...renderAgentSuccessRateTextLines(agentSuccessRate));
+    }
     const contextPackGeneration = report.contextPackGeneration ?? null;
     if (contextPackGeneration !== null) {
         pushSection(lines, "Context Pack Generation");

package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js

@@ -0,0 +1,135 @@
+import { lstat, mkdir, open, readdir, realpath } from "node:fs/promises";
+import path from "node:path";
+import { campaignAssert, campaignHash, canonicalCampaignJson } from "../../experiments/plugins/agentSuccessRate/campaignIdentity.js";
+import { validateCampaignOutputPath } from "../../experiments/plugins/agentSuccessRate/campaignCheckpointStore.js";
+import { readCampaignProgressScientificFamily } from "../../experiments/plugins/agentSuccessRate/campaignProgress.js";
+import { agentSuccessRatePlugin, mapAgentSuccessRateToRun } from "../../experiments/plugins/agentSuccessRate/plugin.js";
+import { writePluginExperimentReports } from "./writePluginExperimentReports.js";
+import { buildPluginExperimentReport } from "./buildPluginExperimentReport.js";
+import { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
+import { renderPluginExperimentReportHtml } from "./renderPluginExperimentReportHtml.js";
+import { buildAgentSuccessCampaignProgressReport } from "./buildAgentSuccessCampaignProgressReport.js";
+import { renderAgentSuccessCampaignProgressText } from "./renderAgentSuccessCampaignProgressText.js";
+import { renderAgentSuccessCampaignProgressHtml } from "./renderAgentSuccessCampaignProgressHtml.js";
+async function plain(p, kind) {
+    const s = await lstat(p);
+    campaignAssert(!s.isSymbolicLink() && (kind === "file" ? s.isFile() : s.isDirectory()) && path.resolve(await realpath(p)).toLowerCase() === path.resolve(p).toLowerCase(), "PROGRESS_UNSAFE_PATH");
+}
+async function writeExclusive(p, content) {
+    const h = await open(p, "wx");
+    try {
+        await h.writeFile(content, "utf8");
+        await h.sync();
+    }
+    finally {
+        await h.close();
+    }
+}
+async function readExpected(p, expected, code) {
+    await plain(p, "file");
+    const length = Buffer.byteLength(expected);
+    campaignAssert(length <= 64 * 1024 * 1024, "PROGRESS_DATA_BOUND");
+    const h = await open(p, "r");
+    try {
+        const before = await h.stat();
+        campaignAssert(before.isFile() && before.size === length, code);
+        const buffer = Buffer.alloc(length + 1);
+        let size = 0;
+        for (;;) {
+            const n = await h.read(buffer, size, buffer.length - size, size);
+            size += n.bytesRead;
+            if (!n.bytesRead || size === buffer.length)
+                break;
+        }
+        campaignAssert(size === length && (await h.stat()).size === length && buffer.subarray(0, size).toString("utf8") === expected, code);
+    }
+    finally {
+        await h.close();
+    }
+}
+async function owned(parent, name, owner) {
+    const p = path.join(parent, name);
+    let created = false;
+    try {
+        await mkdir(p);
+        created = true;
+    }
+    catch (e) {
+        if (e.code !== "EEXIST")
+            throw e;
+    }
+    await plain(p, "directory");
+    if (created)
+        await writeExclusive(path.join(p, "owner.json"), owner);
+    await readExpected(path.join(p, "owner.json"), owner, "PROGRESS_OWNER_MISMATCH");
+    return p;
+}
+/** Revision-specific files are exclusive. The manifest is published last and verified before returning success. */
+export async function writeAgentSuccessCampaignProgressReport(args) {
+    const model = buildAgentSuccessCampaignProgressReport(args.snapshot, args.boundary);
+    await validateCampaignOutputPath(args.boundary);
+    const root = path.resolve(args.boundary.outputRoot);
+    await plain(root, "directory");
+    const family = await owned(root, "campaign-progress-v1", "agent-success-rate-campaign-progress-v1");
+    const identity = await owned(family, model.campaignIdentity, `agent-success-rate-campaign-progress-v1:${model.campaignIdentity}`);
+    // Ownership blockers can change without a checkpoint transition. Keep both projections immutable at that revision.
+    const modelDigest = campaignHash(canonicalCampaignJson(model));
+    const revision = `revision-${String(model.checkpointGeneration).padStart(10, "0")}-${model.checkpointRevision}-${modelDigest.slice(0, 16)}`;
+    const outDir = path.join(identity, revision);
+    const scientific = model.state === "complete" ? await readCampaignProgressScientificFamily(args.snapshot, args.boundary) : null;
+    const run = scientific ? mapAgentSuccessRateToRun({ runId: scientific.execution.runId, startedAt: scientific.execution.startedAt, completedAt: scientific.execution.completedAt,
+        target: { kind: "self", targetRoot: "[redacted]", toolRoot: "[redacted]", packageName: "@dailephd/my-dev-kit-lab", packageVersion: "0.9.0", hasPackageJson: true, hasLockfile: true, branch: null, commit: null, hasGit: false, isSelf: true },
+        caseEvidence: scientific.execution.cases.map(c => ({ ...c, caseName: c.caseId })), analysis: scientific.analysis.analysis, patchArtifactPaths: [],
+        providerId: scientific.execution.realAgent?.providerId, ...(scientific.execution.realAgent ? { realSettings: { timeoutMs: scientific.execution.realAgent.timeoutMs, repairAttempts: scientific.execution.realAgent.repairAttempts ?? 0 } } : {}) }) : null;
+    if (run)
+        Object.assign(run, { agentSuccessCampaignProgress: model });
+    const files = [
+        { name: "agent-success-rate-campaign-progress.json", content: `${JSON.stringify(model, null, 2)}\n`, phase: "json-written" },
+        { name: "agent-success-rate-campaign-progress.txt", content: renderAgentSuccessCampaignProgressText(model), phase: "text-written" },
+        { name: "agent-success-rate-campaign-progress.html", content: renderAgentSuccessCampaignProgressHtml(model), phase: "html-written" }
+    ];
+    if (run) {
+        const built = buildPluginExperimentReport({ run, plugin: agentSuccessRatePlugin.metadata, outputRoot: outDir, generatedAt: run.completedAt });
+        const report = { ...built, metadata: { ...built.metadata, outputRoot: "[redacted]" } };
+        files.push({ name: "report.json", content: `${JSON.stringify({ report, outputPaths: { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" } }, null, 2)}\n`, phase: "json-written" }, { name: "report.txt", content: renderPluginExperimentReportText(report), phase: "text-written" }, { name: "report.html", content: renderPluginExperimentReportHtml(report), phase: "html-written" });
+    }
+    const marker = canonicalCampaignJson({ schemaVersion: "agent-success-rate-campaign-progress-publication-v1", campaignIdentity: model.campaignIdentity, checkpointGeneration: model.checkpointGeneration, checkpointRevision: model.checkpointRevision, modelDigest, files: files.map(f => ({ name: f.name, sha256: campaignHash(f.content), bytes: Buffer.byteLength(f.content) })) });
+    let created = false;
+    try {
+        await mkdir(outDir);
+        created = true;
+    }
+    catch (e) {
+        if (e.code !== "EEXIST")
+            throw e;
+    }
+    await plain(outDir, "directory");
+    if (created) {
+        for (const file of files.slice(0, 3)) {
+            await writeExclusive(path.join(outDir, file.name), file.content);
+            await args.failureHook?.(file.phase);
+        }
+        // Exclusive directory reservation owns these ordinary report names; the existing writer remains their owner.
+        if (run)
+            await writePluginExperimentReports({ run, plugin: agentSuccessRatePlugin.metadata, outputRoot: outDir, generatedAt: run.completedAt, redactOutputRoot: true, exclusive: true });
+        for (const f of files)
+            await readExpected(path.join(outDir, f.name), f.content, "PROGRESS_SNAPSHOT_INTEGRITY");
+        // The legacy writer owns report rendering. Flush its campaign-only output before committing the group marker.
+        for (const f of files.slice(3)) {
+            const h = await open(path.join(outDir, f.name), "r+");
+            try {
+                await h.sync();
+            }
+            finally {
+                await h.close();
+            }
+        }
+        await args.failureHook?.("before-manifest");
+        await writeExclusive(path.join(outDir, "manifest.json"), marker);
+    }
+    campaignAssert((await readdir(outDir)).sort().join(",") === [...files.map(f => f.name), "manifest.json"].sort().join(","), "PROGRESS_INCOMPLETE_OR_UNKNOWN_SNAPSHOT");
+    await readExpected(path.join(outDir, "manifest.json"), marker, "PROGRESS_SNAPSHOT_COLLISION");
+    for (const f of files)
+        await readExpected(path.join(outDir, f.name), f.content, "PROGRESS_SNAPSHOT_INTEGRITY");
+    return { model, outputPaths: { outDir, jsonPath: path.join(outDir, files[0].name), textPath: path.join(outDir, files[1].name), htmlPath: path.join(outDir, files[2].name), manifestPath: path.join(outDir, "manifest.json") } };
+}

package/dist/src/report/experiments/writePluginExperimentReports.js

@@ -31,9 +31,10 @@ export async function writePluginExperimentReports(args) {
     const serializedPaths = args.redactOutputRoot
         ? { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" }
         : outputPaths;
-    await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, "utf8");
-    await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
-    await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
+    const writeOptions = args.exclusive ? { encoding: "utf8", flag: "wx" } : "utf8";
+    await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, writeOptions);
+    await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), writeOptions);
+    await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), writeOptions);
     return { report, outputPaths };
 }
 function readString(value) {

package/docs/METRICS.md

@@ -868,3 +868,82 @@ There is no composite score, winner, or ranking. Tests and call relationships ma
 In external-local mode, analysis uses real identities before the privacy projection. Redacted placeholders never enter metric calculations. Durable external-local output has no pack body; the report preview presents redacted identities.
 
 See [COMMANDS.md](COMMANDS.md#context-pack-generation-v082), [WORKFLOWS.md](WORKFLOWS.md#context-pack-generation-experiment), and [ARCHITECTURE.md](ARCHITECTURE.md#context-pack-generation-architecture-v082).
+
+## Agent-success-rate metrics (v0.9.0)
+
+Status: shipped in v0.9.0. The scientific owner of every per-treatment value is `analyzeAgentSuccessTreatment` in `src/experiments/plugins/agentSuccessRate/analysis.ts`; repair aggregation and paired differences are owned by `repairAnalysis.ts`. Values are persisted in `agent-success-rate-analysis.json` (schema `1.0.0`); `agent-success-rate-execution.json` holds the evidence they are computed from. `contextEffectEvaluated` is `false` in deterministic-fixture mode.
+
+### Availability rules
+
+Every metric is typed `available`, `unavailable`, or `not-applicable`, with a unit and, when not available, a reason.
+
+- **Unavailable** means the evidence needed to compute the value does not exist (for example the provider reported no token usage, a check timed out, the patch was not applied, or the baseline was invalid). It is never converted to zero, never counted as a failure, and excluded from rates and means.
+- **Not applicable** means the quantity is undefined by construction (for example a ratio whose denominator is zero, or `repairSucceeded` when no repair attempt ran).
+- **Invalid baseline exclusion:** a treatment whose unmodified baseline is not evaluable (it needs at least one failing task check, all regression checks passing, and no check timing out or erroring) has no task-success verdict and is excluded from rates; this is a benchmark defect, not an agent failure.
+- **Tri-state verdicts:** a verdict is true, false, or unknown. A failed check makes a conjunction false; otherwise a missing, timed-out, or errored check makes it unknown; only complete passing evidence makes it true.
+
+### Task-success metrics
+
+| Metric | Meaning and formula | Source, units, availability |
+|---|---|---|
+| `taskChecksPassed` | Every trusted task check passed on the changed state | boolean from post-edit verification; unknown if any task check is missing, timed out, or errored |
+| `regressionSafe` | Every regression check that passed on the baseline still passes | boolean; only baseline-passing regression checks are considered |
+| `requiredFactsSatisfied` | Every behavior fact marked required has all its verification checks passing | boolean |
+| `protectedIntegrity` | Protected files are byte-intact after the patch | boolean from the protected-integrity comparison |
+| `taskResolved` | `taskChecksPassed` AND `requiredFactsSatisfied` | boolean |
+| `taskSuccess` | `taskResolved` AND `regressionSafe` AND `protectedIntegrity` intact. A patch that was rejected or not applied yields false; an uncomputable verdict yields unavailable | boolean; the only success verdict; agent prose is never evidence |
+| `initialAttemptTaskSuccess` | `taskSuccess` of attempt 1 | boolean; real-agent repair evidence only |
+| `finalTaskSuccess` | `taskSuccess` of the last attempt executed | boolean; equals `taskSuccess` |
+| `repairSucceeded` | `initialAttemptTaskSuccess` is false AND `finalTaskSuccess` is true | boolean; not applicable when no repair attempt ran; unavailable if either verdict is unavailable |
+| `repairSuccessRate` (aggregate) | repaired cases / repair-attempted cases with a determinate repair verdict | ratio; not applicable when no repair attempt ran for the treatment |
+| `taskSuccessRate`, `initialAttemptSuccessRate`, `finalTaskSuccessRate` (aggregates) | successful cases / cases with a determinate verdict for that attempt basis | ratio; unavailable when no case has a determinate verdict |
+
+First versus final attempt: `initialAttemptSuccessRate` and `finalTaskSuccessRate` are separate measurements. A repaired success raises the final rate and never the initial rate. Without repair evidence a single attempt makes the two equal.
+
+### Check, fact, and edit-quality metrics
+
+| Metric | Formula | Numerator / denominator | Notes |
+|---|---|---|---|
+| `taskCheckPassRate` | passed task checks / task checks | `taskCheckPassedCount` / `taskCheckTotalCount` | unavailable unless every task check has a complete result; not applicable with zero checks |
+| `regressionCheckPassRate` | passed regression checks / regression checks | `regressionCheckPassedCount` / `regressionCheckTotalCount` | `regressionFailureCount` counts failed checks that passed on the baseline |
+| `requiredFactCoverage` | satisfied required facts / required facts | `requiredFactsSatisfiedCount` / `requiredFactsTotal` | not applicable with no required facts |
+| `factCoverage` | satisfied facts (required and optional) / all facts | satisfied / total facts | not applicable with no facts |
+| `expectedEditCoverage` | changed files that were expected / expected edit files | `expectedEditFilesChangedCount` / `expectedEditFileCount` | ratio |
+| `editScopePrecision` | changed files inside the allowed edit scope / changed files | `allowedEditChangedFileCount` / `changedFileCount` | not applicable when no file changed |
+| `unexpectedChangedFileCount` | changed files minus changed files inside the allowed scope | count | files outside the allowed scope |
+| `protectedMutationCount` | protected paths found mutated after the patch | count | unavailable if integrity could not be proven; `attemptedProtectedEditCount` counts protected paths a patch tried to edit |
+| `changedFileCount`, `addedFileCount`, `modifiedFileCount`, `deletedFileCount` | counts of captured changed files by status | count | from the changed-file capture of the final attempt |
+| `linesAdded`, `linesDeleted` | sums of per-file additions and deletions | lines | unavailable if any changed file has no line counts (binary); never estimated |
+| `totalChurn` | `linesAdded` + `linesDeleted` | lines | the final patch's churn, not the churn accumulated over repair attempts |
+| `relativeChurn` | `totalChurn` / `baselineTextLineCount` | ratio | unavailable when the baseline text-line count is unknown; not applicable at zero baseline lines |
+
+Edit-quality and blast-radius metrics describe the final attempt only. A treatment that never applied a patch has these metrics unavailable, not zero.
+
+### Time and token metrics
+
+| Metric | Meaning | Source and caveat |
+|---|---|---|
+| `agentDurationMs` | Measured wall-clock duration of the provider attempt | unavailable in deterministic mode and when no provider ran; for repair runs, `firstAttemptProviderDurationMs`, `finalAttemptProviderDurationMs`, and `totalProviderDurationMs` are reported separately |
+| `totalProviderDurationMs` | Sum of provider durations over attempts that ran | unavailable if any invoked attempt was not measured |
+| `agentTotalTokens` (provider token usage) | Provider-reported total token usage for the attempt | unavailable when the provider reported none (source and reliability are recorded); `totalProviderTokens` sums over attempts and is unavailable if any invoked attempt lacks a total |
+| `estimatedContextTokens` | Estimated size of the context supplied to the provider | a size estimate recorded in execution evidence, not provider-reported usage; never substituted for provider tokens |
+| `baselineVerificationDurationMs`, `patchPipelineDurationMs`, `postEditVerificationDurationMs`, `evaluationDurationMs` | Measured durations of the evaluator's own stages | milliseconds; `totalEvaluationDurationMs` sums attempts |
+
+Cumulative versus final: durations and tokens accumulate over all provider attempts (repair costs are included), while edit-quality and blast-radius metrics describe only the final patch.
+
+### Paired comparisons and means
+
+- Aggregate means (`meanTaskCheckPassRate`, `meanEditScopePrecision`, `meanTotalChurn`, and the other `mean*` entries) are unweighted arithmetic means over **matched** cases: cases for which the source metric is available in every treatment. Unavailable values are not converted to zero. A mean with no matched case is unavailable.
+- The matched comparison counts, over cases with a determinate verdict in both treatments, `bothSucceeded`, `onlyRawFullFileSucceeded`, `onlyContextPackSucceeded`, and `neitherSucceeded`; incomplete cases are listed separately with the unavailable treatment IDs. With repair evidence a second matched comparison uses initial-attempt success.
+- Paired differences report the mean of the `raw-full-file` values, the mean of the `context-pack` values, and the mean of `context-pack` minus `raw-full-file` over matched cases.
+- These are **descriptive**. No weighted composite score, ranking, automatic winner, causal claim, or statistical-significance test is produced (`statisticalEffect` is `not-assessed`). The corpus is finite and controlled, so results do not generalize beyond it, and a single run carries no variance estimate.
+- Provider telemetry can be unavailable, and in deterministic-fixture mode both treatments receive the same reference patch, so those results measure the pipeline only.
+
+
+## Campaign progress evidence (v0.9.1 implemented; unreleased)
+
+Token evidence levels are `provider-reported`, `cli-reported`, `parsed-from-output`, `unavailable`, and `estimated`. The original telemetry source is retained. Numeric availability is recorded per token field, so partial measurements remain partial and normalization never fills missing fields or invents totals; unattributed OpenTelemetry evidence is unavailable. Duration provenance is `lab-wall-clock` for valid observed elapsed time and `unavailable` for missing or invalid values; campaign comparison time is Lab-measured wall-clock duration, not provider-native time.
+
+The plugin-specific progress projection is `my-dev-kit-lab-agent-success-rate-campaign-progress-v1`; the writer publishes immutable revision-specific JSON, HTML, and text snapshots. It accounts for selected cases and treatment units, while execution epochs remain recovery history and are excluded from selected-unit counts. Paused provider-limit and uncertain states remain distinct. Partial progress is not a complete scientific analysis artifact.
+
+Evidence quality is metric-specific: task outcomes, token measurements, and duration can have different strong, partial, or unavailable classifications. Strong describes the completeness and provenance of the available evidence; it is not statistical significance. Partial evidence does not mean task failure. Missing token evidence is unavailable, never zero; estimated tokens are not provider usage. Duration is Lab-measured wall-clock time, not provider-native time. Comparisons use matched completed cases and state their denominator and exclusions; the existing paired direction is preserved. These descriptive comparisons establish no causal effect, winner, p-value, or general provider performance.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.8.2",
+  "version": "0.9.1",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {