@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/experiments/plugins/agentSuccessRate/repairPolicy.js

@@ -0,0 +1,88 @@
+/**
+ * Pure repair policy: decides how one evaluated attempt ended and whether a repair may follow. It performs no
+ * filesystem, subprocess or provider work and reads only normalized execution evidence plus already-calculated metrics.
+ */
+/** Categories for which a completed provider attempt produced an evaluable implementation failure. */
+export const AGENT_SUCCESS_REPAIR_ELIGIBLE_CATEGORIES = [
+    "patch-malformed",
+    "patch-policy-rejected",
+    "patch-check-failed",
+    "patch-apply-failed",
+    "task-check-failed",
+    "regression-failed",
+    "required-behavior-unsatisfied"
+];
+function booleanValue(metric) {
+    return metric !== undefined && metric.availability === "available" && typeof metric.value === "boolean" ? metric.value : null;
+}
+/** Classifies an attempt. Infrastructure and provider-health outcomes are never eligible, regardless of task success. */
+export function classifyAgentSuccessAttempt(evidence, metrics) {
+    const taskSuccess = booleanValue(metrics.taskSuccess);
+    const result = (category) => ({
+        category,
+        repairEligible: AGENT_SUCCESS_REPAIR_ELIGIBLE_CATEGORIES.includes(category),
+        taskSuccess
+    });
+    const real = evidence.realAgent;
+    if (evidence.availability === "infrastructure-failure")
+        return result("infrastructure-failure");
+    if (evidence.availability === "baseline-invalid" || evidence.baselineAssessment === null || !evidence.baselineAssessment.evaluable)
+        return result("baseline-invalid");
+    if (real === undefined)
+        return result("evidence-incomplete");
+    if (real.context.availability === "unavailable" && !real.providerInvoked)
+        return result("context-unavailable");
+    if (!real.providerInvoked)
+        return result("provider-not-invoked");
+    switch (real.providerStatus) {
+        case "agent-unavailable":
+            return result("provider-unavailable");
+        case "agent-limit-reached":
+            return result("provider-limit-reached");
+        case "timeout":
+            return result("provider-timeout");
+        case "completed":
+            break;
+        default:
+            return result("provider-failed");
+    }
+    if (!real.finalAnswerAvailable || !evidence.patch.attempted)
+        return result("provider-empty-answer");
+    if (!evidence.cleanup.removed || (real.cwdCleanup.attempted && !real.cwdCleanup.removed))
+        return result("cleanup-failed");
+    if (evidence.protectedIntegrity.status === "unproven")
+        return result("integrity-unproven");
+    if (evidence.availability !== "complete")
+        return result("evidence-incomplete");
+    switch (evidence.patch.outcome) {
+        case "parse-failure":
+            return result("patch-malformed");
+        case "policy-rejection":
+            return result("patch-policy-rejected");
+        case "git-check-failure":
+            return result("patch-check-failed");
+        case "git-apply-failure":
+            return result("patch-apply-failed");
+        case "success":
+            break;
+        default:
+            return result("evidence-incomplete");
+    }
+    if (evidence.protectedIntegrity.status === "mutated")
+        return result("integrity-mutated");
+    if (taskSuccess === true)
+        return result("none");
+    if (taskSuccess === null)
+        return result("evidence-incomplete");
+    if (booleanValue(metrics.taskChecksPassed) === false)
+        return result("task-check-failed");
+    if (booleanValue(metrics.regressionSafe) === false)
+        return result("regression-failed");
+    if (booleanValue(metrics.requiredFactsSatisfied) === false)
+        return result("required-behavior-unsatisfied");
+    return result("task-check-failed");
+}
+/** A repair runs only after an eligible failure, and only while the allowance and the hard total bound both remain. */
+export function shouldRepairAgentSuccessAttempt(args) {
+    return args.classification.repairEligible && args.classification.taskSuccess === false && args.attemptsExecuted <= args.repairAttemptsAllowed && args.attemptsExecuted < 3;
+}

package/dist/src/experiments/plugins/agentSuccessRate/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/warmIndexReuse/plugin.js

@@ -42,9 +42,8 @@ export const warmIndexReusePlugin = {
         const startedAt = context.startedAt.toISOString();
         // A campaign preset selects campaign-aware case selection and real-agent evaluation for
         // exactly one provider; its absence preserves the legacy fake-agent path unchanged. Real-agent
-        // campaign execution is still guarded at the public experiment-run command (v0.5.2 Batch 3):
-        // this plugin-level path is exercised by programmatic/test callers only until Batch 4
-        // implements campaign-accurate reporting.
+        // execution stays explicitly gated: the config requires includeRealAgents for a preset, and the
+        // public experiment-run command rejects incompatible options before this path is reached.
         const preset = context.config.campaignPreset ? getWarmIndexCampaignPreset(context.config.campaignPreset) : undefined;
         const cases = preset
             ? selectWarmIndexCampaignCases(readCasesInput(context.inputs), preset, context.config)

package/dist/src/report/experiments/agentSuccessRateReportModel.js

@@ -0,0 +1,6 @@
+/**
+ * Typed agent-success-rate report section. Every value here is copied from the plugin's already-calculated analysis and
+ * execution evidence: the report layer rescored nothing, reran no check, reparsed no patch and read no reference patch.
+ * It holds no prompt, source-file or patch body and no raw command output.
+ */
+export const AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-rate-report-v1";

package/dist/src/report/experiments/buildAgentSuccessRateReport.js

@@ -0,0 +1,255 @@
+import { attemptEvidenceOf } from "../../experiments/plugins/agentSuccessRate/attemptEvidence.js";
+import { AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../../experiments/plugins/agentSuccessRate/metadata.js";
+import { AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION } from "./agentSuccessRateReportModel.js";
+export const AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT = "The fixture validates the patch evaluation pipeline and benchmark corpus. It does not measure a coding agent or the effect of context selection.";
+export const AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT = "The observed outcomes are a descriptive matched comparison of two source-context treatments under the selected provider.";
+const MAX_TEXT = 300;
+const MAX_CHANGED_FILES = 20;
+const bound = (text) => {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    return normalized.length > MAX_TEXT ? `${normalized.slice(0, MAX_TEXT)}...` : normalized;
+};
+/** Copies a plugin metric. Availability, value and reason are carried through untouched. */
+function copyMetric(metric, fallbackUnit = "count") {
+    if (!metric)
+        return { availability: "unavailable", value: null, unit: fallbackUnit, reason: "the measurement was not recorded." };
+    return { availability: metric.availability, value: metric.value, unit: metric.unit, reason: metric.reason === null ? null : bound(metric.reason) };
+}
+function copyTotal(total) {
+    return {
+        basis: total.basis,
+        contributingCaseCount: total.contributingCaseCount,
+        availableCaseCount: total.availableCaseCount,
+        unavailableCaseCount: total.unavailableCaseCount,
+        sumOfAvailable: total.sumOfAvailable,
+        total: copyMetric(total.total)
+    };
+}
+function copyDifference(entry) {
+    return {
+        id: entry.id,
+        basis: entry.basis,
+        matchedCaseIds: [...entry.matchedCaseIds],
+        rawFullFileMean: copyMetric(entry.rawFullFileMean),
+        contextPackMean: copyMetric(entry.contextPackMean),
+        meanDifference: copyMetric(entry.meanDifference)
+    };
+}
+/**
+ * A relative, forward-slash artifact reference with no traversal, drive, scheme or absolute form. Anything else is
+ * withheld rather than rendered, so a report can never point outside the experiment output directory.
+ */
+export function safeAgentSuccessArtifactReference(reference) {
+    if (typeof reference !== "string" || reference.length === 0 || reference.length > 400)
+        return null;
+    if (reference.includes("\\") || reference.includes("\0") || reference.startsWith("/") || /^[A-Za-z]:/.test(reference) || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(reference))
+        return null;
+    const segments = reference.split("/");
+    if (segments.some((segment) => segment === "" || segment === "." || segment === ".."))
+        return null;
+    return reference;
+}
+function patchOutcome(evidence) {
+    return evidence.patch.outcome ?? (evidence.patch.attempted ? "unrecorded" : "not-attempted");
+}
+function verificationAvailability(evidence) {
+    return evidence.postEditVerification !== null ? "available" : "unavailable";
+}
+export function buildAgentSuccessRateReport(run) {
+    if (run.pluginId !== AGENT_SUCCESS_RATE_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid agent-success-rate report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== AGENT_SUCCESS_RATE_TREATMENT_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.treatmentId !== AGENT_SUCCESS_RATE_TREATMENT_IDS[position]));
+    if (inconsistent) {
+        throw new Error("Invalid agent-success-rate report source: execution evidence and analysis are inconsistent.");
+    }
+    const realMode = analysis.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE;
+    const metadata = (run.metadata ?? {});
+    const repairAttempts = realMode && typeof metadata.repairAttempts === "number" ? metadata.repairAttempts : realMode ? 0 : null;
+    const timeoutMs = realMode && typeof metadata.timeoutMs === "number" ? metadata.timeoutMs : null;
+    const cases = analysis.cases.map((entry, caseIndex) => {
+        const evidence = execution[caseIndex];
+        return {
+            caseId: entry.caseId,
+            caseName: bound(evidence.caseName),
+            benchmarkProject: entry.benchmarkProject,
+            taskLocality: evidence.taskLocality,
+            treatments: entry.treatments.map((treatment, treatmentIndex) => {
+                const treatmentEvidence = evidence.treatments[treatmentIndex];
+                const repair = treatment.repair;
+                const changed = treatmentEvidence.change?.changedFiles.map((file) => file.relativePath).filter((file) => safeAgentSuccessArtifactReference(file) !== null) ?? null;
+                const errors = attemptEvidenceOf(treatmentEvidence).flatMap((attempt, index) => attempt.errors.slice(0, 10).map((error) => ({ attemptNumber: index + 1, code: bound(error.code), message: bound(error.message) })));
+                return {
+                    treatmentId: treatment.treatmentId,
+                    executionStatus: treatment.executionStatus,
+                    evidenceAvailability: treatment.evidenceAvailability,
+                    initialTaskSuccess: copyMetric(repair ? repair.initialAttemptTaskSuccess : treatment.metrics.taskSuccess, "boolean"),
+                    finalTaskSuccess: copyMetric(treatment.metrics.taskSuccess, "boolean"),
+                    attemptCount: repair ? repair.attemptCount : 1,
+                    repairAttemptCount: repair ? repair.repairAttemptCount : 0,
+                    repairSucceeded: repair
+                        ? copyMetric(repair.repairSucceeded, "boolean")
+                        : { availability: "not-applicable", value: null, unit: "boolean", reason: "deterministic-fixture mode has no repair." },
+                    taskCheckPassRate: copyMetric(treatment.metrics.taskCheckPassRate, "ratio"),
+                    regressionCheckPassRate: copyMetric(treatment.metrics.regressionCheckPassRate, "ratio"),
+                    requiredFactCoverage: copyMetric(treatment.metrics.requiredFactCoverage, "ratio"),
+                    expectedEditCoverage: copyMetric(treatment.metrics.expectedEditCoverage, "ratio"),
+                    editScopePrecision: copyMetric(treatment.metrics.editScopePrecision, "ratio"),
+                    unexpectedChangedFiles: copyMetric(treatment.metrics.unexpectedChangedFileCount, "count"),
+                    protectedMutations: copyMetric(treatment.metrics.protectedMutationCount, "count"),
+                    changedFileCount: copyMetric(treatment.metrics.changedFileCount, "count"),
+                    changedFiles: changed === null || treatment.metrics.changedFileCount.availability !== "available"
+                        ? null
+                        : { items: changed.slice(0, MAX_CHANGED_FILES), totalCount: changed.length, omittedCount: Math.max(0, changed.length - MAX_CHANGED_FILES) },
+                    totalChurn: copyMetric(treatment.metrics.totalChurn, "lines"),
+                    relativeChurn: copyMetric(treatment.metrics.relativeChurn, "ratio"),
+                    finalAttemptProviderDurationMs: copyMetric(treatment.metrics.agentDurationMs, "ms"),
+                    finalAttemptProviderTokens: copyMetric(treatment.metrics.agentTotalTokens, "tokens"),
+                    totalProviderDurationMs: repair ? copyMetric(repair.totalProviderDurationMs, "ms") : null,
+                    totalProviderTokens: repair ? copyMetric(repair.totalProviderTokens, "tokens") : null,
+                    errors: errors.slice(0, 20)
+                };
+            })
+        };
+    });
+    const treatments = analysis.aggregates.map((aggregate) => {
+        const repair = aggregate.repair;
+        return {
+            treatmentId: aggregate.treatmentId,
+            evaluableCases: repair ? repair.finalEvaluableCount : aggregate.evaluableCaseCount,
+            initialSuccessfulCases: repair ? repair.initialAttemptSuccessfulCount : aggregate.successfulCaseCount,
+            initialSuccessRate: copyMetric(aggregate.initialAttemptSuccessRate, "ratio"),
+            finalSuccessfulCases: repair ? repair.finalSuccessfulCount : aggregate.successfulCaseCount,
+            finalSuccessRate: copyMetric(repair ? repair.finalTaskSuccessRate : aggregate.taskSuccessRate, "ratio"),
+            repair: repair
+                ? {
+                    repairEligibleCases: repair.repairEligibleCaseCount,
+                    repairAttemptedCases: repair.repairAttemptedCaseCount,
+                    repairAttempts: repair.totalRepairAttemptCount,
+                    repairedCases: repair.repairedCaseCount,
+                    repairSuccessRate: copyMetric(repair.repairSuccessRate, "ratio"),
+                    meanAttemptsPerEvaluableCase: copyMetric(repair.meanAttemptsPerEvaluableCase, "count"),
+                    providerDurationMs: repair.providerDurationMs.map(copyTotal),
+                    providerTokens: repair.providerTokens.map(copyTotal)
+                }
+                : null
+        };
+    });
+    const repairHistory = realMode
+        ? analysis.cases.flatMap((entry, caseIndex) => entry.treatments.flatMap((treatment, treatmentIndex) => {
+            const repair = treatment.repair;
+            const evidence = execution[caseIndex].treatments[treatmentIndex];
+            if (!repair || !evidence.attempts)
+                return [];
+            return [
+                {
+                    caseId: entry.caseId,
+                    treatmentId: treatment.treatmentId,
+                    attempts: repair.attempts.map((attempt, index) => {
+                        const attemptEvidence = evidence.attempts[index];
+                        const inner = attemptEvidence.evidence;
+                        return {
+                            attemptNumber: attempt.attemptNumber,
+                            providerStatus: attempt.providerStatus,
+                            patchOutcome: patchOutcome(inner),
+                            verificationAvailability: verificationAvailability(inner),
+                            taskSuccess: copyMetric(attempt.metrics.taskSuccess, "boolean"),
+                            failureCategory: attempt.failureCategory,
+                            repairEligible: attempt.repairEligible,
+                            providerDurationMs: copyMetric(attempt.metrics.agentDurationMs, "ms"),
+                            providerTokens: copyMetric(attempt.metrics.agentTotalTokens, "tokens"),
+                            tokenAvailability: attempt.metrics.agentTotalTokens.availability === "available" ? "available" : "unavailable",
+                            proposedPatchPath: safeAgentSuccessArtifactReference(inner.proposedPatchPath),
+                            appliedPatchPath: safeAgentSuccessArtifactReference(inner.appliedPatchPath),
+                            agentArtifactDirectory: safeAgentSuccessArtifactReference(inner.realAgent?.agentArtifactDirectory ?? null)
+                        };
+                    })
+                }
+            ];
+        }))
+        : [];
+    const outcomeAvailability = analysis.cases.flatMap((entry) => entry.treatments.map((treatment) => treatment.metrics.taskSuccess.availability));
+    const evaluable = outcomeAvailability.filter((availability) => availability === "available").length;
+    const total = outcomeAvailability.length;
+    const unavailable = total - evaluable;
+    const notes = [];
+    if (unavailable > 0)
+        notes.push(`${unavailable} of ${total} treatment outcome(s) have no determinate task-success verdict; they are excluded from rates and never counted as failures.`);
+    if (realMode && analysis.comparison && analysis.comparison.incompleteCases.length > 0) {
+        notes.push(`${analysis.comparison.incompleteCases.length} case(s) are excluded from matched comparisons because a treatment verdict is unavailable.`);
+    }
+    const comparison = analysis.comparison
+        ? (() => {
+            const repairComparison = analysis.repairComparison;
+            const finalMatched = analysis.comparison.matchedCaseIds.length;
+            const evaluableComparison = repairComparison ? repairComparison.interpretation.evaluable : finalMatched > 0;
+            const finalPairs = analysis.comparison.pairedOutcomes;
+            const differs = finalPairs.onlyRawFullFileSucceeded + finalPairs.onlyContextPackSucceeded > 0;
+            return {
+                comparisonEvaluable: evaluableComparison,
+                observedOutcomeDifference: repairComparison ? repairComparison.interpretation.observedOutcomeDifference : !evaluableComparison ? "not-evaluable" : differs ? "observed" : "not-observed",
+                statisticalEffect: "not-assessed",
+                finalMatchedCaseIds: [...analysis.comparison.matchedCaseIds],
+                finalIncompleteCases: analysis.comparison.incompleteCases.map((entry) => ({ caseId: entry.caseId, unavailableTreatmentIds: [...entry.unavailableTreatmentIds] })),
+                finalPairedOutcomes: { ...finalPairs },
+                initialMatchedCaseIds: repairComparison ? [...repairComparison.initialAttempt.matchedCaseIds] : null,
+                initialIncompleteCases: repairComparison ? repairComparison.initialAttempt.incompleteCases.map((entry) => ({ caseId: entry.caseId, unavailableTreatmentIds: [...entry.unavailableTreatmentIds] })) : null,
+                initialPairedOutcomes: repairComparison ? { ...repairComparison.initialAttempt.pairedOutcomes } : null,
+                pairedDifferences: repairComparison ? repairComparison.pairedDifferences.map(copyDifference) : []
+            };
+        })()
+        : null;
+    const overall = total === 0 || evaluable === 0 ? "unavailable" : unavailable === 0 ? "complete" : "partial";
+    return {
+        schemaVersion: AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION,
+        identity: {
+            runId: run.runId,
+            pluginId: run.pluginId,
+            pluginSchemaVersion: "1.0.0",
+            runStatus: run.status,
+            executionMode: analysis.executionMode,
+            providerId: typeof metadata.providerId === "string" ? metadata.providerId : null,
+            timeoutMs,
+            repairAttempts,
+            maxAttemptsPerTreatment: repairAttempts === null ? 1 : 1 + repairAttempts,
+            treatmentOrder: [...analysis.treatmentOrder],
+            caseCount: analysis.caseCount,
+            treatmentOutcomeCount: total
+        },
+        scientificStatement: realMode ? AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT : AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT,
+        limitations: [
+            ...analysis.limitations.map(bound),
+            ...(realMode
+                ? [
+                    "The corpus is finite and a single run supports no generalization.",
+                    "Provider output can vary between runs, and provider usage or availability can differ between treatments.",
+                    "No causal or statistical-significance claim is made; no weighted composite score and no automatic best treatment exist."
+                ]
+                : [])
+        ],
+        resultAvailability: { overall, evaluableTreatmentOutcomes: evaluable, unavailableTreatmentOutcomes: unavailable, totalTreatmentOutcomes: total, notes },
+        treatments,
+        comparison,
+        cases,
+        repairHistory,
+        warnings: run.warnings.map((warning) => ({ code: bound(warning.code), message: bound(warning.message) })),
+        artifacts: run.artifacts.map((artifact) => ({
+            id: bound(artifact.id),
+            label: bound(artifact.label),
+            path: safeAgentSuccessArtifactReference(artifact.path),
+            caseId: artifact.caseId ?? null,
+            variantId: artifact.variantId ?? null
+        }))
+    };
+}

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -6,6 +6,7 @@ import { buildContextWindowScalingReport } from "./buildContextWindowScalingRepo
 import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
 import { buildRetrievalQueryStrategyComparisonReport } from "./buildRetrievalQueryStrategyComparisonReport.js";
 import { buildContextPackGenerationReport } from "./buildContextPackGenerationReport.js";
+import { AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT, AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT, buildAgentSuccessRateReport } from "./buildAgentSuccessRateReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
@@ -15,6 +16,8 @@ const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregat
 const RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 // Same for context-pack-generation: the typed section presents execution evidence and analysis.
 const CONTEXT_PACK_GENERATION_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
+// Same for agent-success-rate: the typed section presents execution evidence and analysis.
+const AGENT_SUCCESS_RATE_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -30,6 +33,7 @@ export function buildPluginExperimentReport(args) {
     const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
     const retrievalQueryStrategyComparison = buildRetrievalQueryStrategyComparisonReport(args.run);
     const contextPackGeneration = buildContextPackGenerationReport(args.run, args.contextPacks);
+    const agentSuccessRate = buildAgentSuccessRateReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -54,6 +58,11 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (agentSuccessRate) {
+        for (const key of AGENT_SUCCESS_RATE_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -83,8 +92,9 @@ export function buildPluginExperimentReport(args) {
         retrievalPrecisionRecall,
         retrievalQueryStrategyComparison,
         contextPackGeneration,
+        agentSuccessRate,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration, agentSuccessRate),
         rawRun,
     };
 }
@@ -163,7 +173,22 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration = null) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration = null, agentSuccessRate = null) {
+    if (agentSuccessRate) {
+        const real = agentSuccessRate.identity.executionMode === "real-agent";
+        const [raw, pack] = agentSuccessRate.treatments;
+        const rate = (treatment, kind) => {
+            const metric = kind === "initial" ? treatment?.initialSuccessRate : treatment?.finalSuccessRate;
+            return metric && metric.availability === "available" && typeof metric.value === "number" ? metric.value.toFixed(4) : "unavailable";
+        };
+        const summary = real
+            ? `${AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT} Provider ${agentSuccessRate.identity.providerId ?? "unknown"} ran ${agentSuccessRate.identity.caseCount} case(s) with up to ${agentSuccessRate.identity.maxAttemptsPerTreatment} attempt(s) per treatment. Initial-attempt success rate: raw-full-file ${rate(raw, "initial")}, context-pack ${rate(pack, "initial")}; final success rate: raw-full-file ${rate(raw, "final")}, context-pack ${rate(pack, "final")}. No treatment is declared best and no statistical or causal claim is made.`
+            : `${AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT} ${agentSuccessRate.identity.caseCount} case(s) were evaluated for each treatment identity.`;
+        return {
+            summary,
+            recommendedNextStep: "Review per-case results, the repair history and the scientific limitations; unavailable evidence is never counted as a failure or as zero."
+        };
+    }
     if (contextPackGeneration) {
         const overall = contextPackGeneration.scopes.find((scope) => scope.scopeId === "overall");
         const saved = overall?.tokenSavings;

package/dist/src/report/experiments/index.js

@@ -33,6 +33,10 @@ export { buildContextPackGenerationReport, CONTEXT_PACK_GENERATION_LIMITATIONS }
 export { loadContextPackArtifacts } from "./loadContextPackArtifacts.js";
 export { renderContextPackGenerationHtml } from "./renderContextPackGenerationHtml.js";
 export { renderContextPackGenerationTextLines } from "./renderContextPackGenerationText.js";
+export * from "./agentSuccessRateReportModel.js";
+export { buildAgentSuccessRateReport, safeAgentSuccessArtifactReference } from "./buildAgentSuccessRateReport.js";
+export { renderAgentSuccessRateHtml } from "./renderAgentSuccessRateHtml.js";
+export { renderAgentSuccessRateTextLines } from "./renderAgentSuccessRateText.js";
 export * from "./retrievalQueryStrategyComparisonReportModel.js";
 export { buildRetrievalQueryStrategyComparisonReport, RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS } from "./buildRetrievalQueryStrategyComparisonReport.js";
 export { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";

package/dist/src/report/experiments/renderAgentSuccessRateHtml.js

@@ -0,0 +1,169 @@
+import { AGENT_SUCCESS_BASIS_LABEL, formatAgentSuccessMetric, formatAgentSuccessPairs, formatAgentSuccessTotal } from "./renderAgentSuccessRateText.js";
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+/**
+ * HTML rendering of the typed section. All dynamic content is escaped, artifact references are plain text (never links
+ * or embedded files), and no value is recalculated here.
+ */
+export function renderAgentSuccessRateHtml(report) {
+    if (!report)
+        return "";
+    const realMode = report.identity.executionMode === "real-agent";
+    const m = formatAgentSuccessMetric;
+    const parts = [];
+    parts.push(`<section id="agent-success-rate">
+    <h2>Agent Success Rate</h2>
+    <h3>Experiment identity</h3>
+    ${table(["Field", "Value"], [
+        ["Run ID", report.identity.runId],
+        ["Plugin", `${report.identity.pluginId} (schema ${report.identity.pluginSchemaVersion})`],
+        ["Run status", report.identity.runStatus],
+        ["Cases", String(report.identity.caseCount)],
+        ["Treatments", report.identity.treatmentOrder.join(", ")],
+        ["Treatment outcomes", String(report.identity.treatmentOutcomeCount)]
+    ])}
+    <h3>Execution mode and provider</h3>
+    ${table(["Field", "Value"], [
+        ["Execution mode", report.identity.executionMode],
+        ["Provider", report.identity.providerId ?? "none (no coding agent was invoked)"],
+        ["Provider timeout per attempt", report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`],
+        ["Repair attempts allowed", report.identity.repairAttempts === null ? "not applicable" : String(report.identity.repairAttempts)],
+        ["Maximum attempts per treatment", String(report.identity.maxAttemptsPerTreatment)]
+    ])}
+    <h3>Scientific limitations</h3>
+    <p><strong>${escapeHtml(report.scientificStatement)}</strong></p>
+    <ul>${report.limitations.map((limitation) => `<li>${escapeHtml(limitation)}</li>`).join("")}</ul>
+    <h3>Overall result availability</h3>
+    ${table(["Field", "Value"], [
+        ["Overall", report.resultAvailability.overall],
+        ["Outcomes with a determinate verdict", `${report.resultAvailability.evaluableTreatmentOutcomes} of ${report.resultAvailability.totalTreatmentOutcomes}`],
+        ["Outcomes without a determinate verdict", String(report.resultAvailability.unavailableTreatmentOutcomes)]
+    ])}
+    ${report.resultAvailability.notes.length > 0 ? `<ul>${report.resultAvailability.notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")}</ul>` : ""}
+  </section>`);
+    const treatmentRows = report.treatments.map((treatment) => [
+        treatment.treatmentId,
+        String(treatment.evaluableCases),
+        String(treatment.initialSuccessfulCases),
+        m(treatment.initialSuccessRate),
+        String(treatment.finalSuccessfulCases),
+        m(treatment.finalSuccessRate),
+        treatment.repair ? String(treatment.repair.repairAttempts) : "not applicable",
+        treatment.repair ? String(treatment.repair.repairedCases) : "not applicable"
+    ]);
+    const comparison = report.comparison;
+    parts.push(`<section id="agent-success-rate-comparison">
+    <h2>Treatment Comparison</h2>
+    ${table(["Treatment", "Evaluable cases", "Initial successful cases", "Initial success rate", "Final successful cases", "Final success rate", "Repair attempts", "Repaired cases"], treatmentRows)}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .map((treatment) => `<p class="muted">${escapeHtml(treatment.treatmentId)}: repair-eligible cases ${treatment.repair.repairEligibleCases}; cases with a repair ${treatment.repair.repairAttemptedCases}; repair success rate ${escapeHtml(m(treatment.repair.repairSuccessRate))}; mean attempts per evaluable case ${escapeHtml(m(treatment.repair.meanAttemptsPerEvaluableCase))}.</p>`)
+        .join("")}
+    ${comparison
+        ? `<h3>Matched comparison (descriptive; no winner is declared)</h3>
+    ${table(["Statement", "Value"], [
+            ["Comparison evaluable", comparison.comparisonEvaluable ? "yes" : "no"],
+            ["Observed outcome difference between treatments", comparison.observedOutcomeDifference],
+            ["Statistical or causal effect", `${comparison.statisticalEffect} (no significance or causal claim is made)`],
+            ["Final matched cases", comparison.finalMatchedCaseIds.join(", ") || "none"],
+            ["Final paired outcomes", formatAgentSuccessPairs(comparison.finalPairedOutcomes)],
+            ["Initial-attempt matched cases", comparison.initialMatchedCaseIds ? comparison.initialMatchedCaseIds.join(", ") || "none" : "not applicable"],
+            ["Initial-attempt paired outcomes", formatAgentSuccessPairs(comparison.initialPairedOutcomes)]
+        ])}
+    ${comparison.pairedDifferences.length > 0
+            ? table(["Measurement", "Basis", "Matched cases", "raw-full-file mean", "context-pack mean", "context-pack minus raw-full-file"], comparison.pairedDifferences.map((difference) => [
+                difference.id,
+                AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis,
+                String(difference.matchedCaseIds.length),
+                m(difference.rawFullFileMean),
+                m(difference.contextPackMean),
+                m(difference.meanDifference)
+            ]))
+            : ""}`
+        : `<p class="muted">Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).</p>`}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cases">
+    <h2>Per-Case Results</h2>
+    ${table(["Case", "Project", "Locality", "Treatment", "Status", "Initial task success", "Final task success", "Attempts", "Task-check pass rate", "Regression-check pass rate", "Required-fact coverage"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality,
+        treatment.treatmentId,
+        `${treatment.executionStatus} / ${treatment.evidenceAvailability}`,
+        m(treatment.initialTaskSuccess),
+        m(treatment.finalTaskSuccess),
+        String(treatment.attemptCount),
+        m(treatment.taskCheckPassRate),
+        m(treatment.regressionCheckPassRate),
+        m(treatment.requiredFactCoverage)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-repair-history">
+    <h2>Per-Attempt Repair History</h2>
+    ${!realMode || report.repairHistory.length === 0
+        ? `<p class="muted">${realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts."}</p>`
+        : table(["Case", "Treatment", "Attempt", "Provider status", "Patch outcome", "Verification", "Task success", "Outcome", "Duration", "Tokens", "Artifacts"], report.repairHistory.flatMap((history) => history.attempts.map((attempt) => [
+            history.caseId,
+            history.treatmentId,
+            String(attempt.attemptNumber),
+            attempt.providerStatus,
+            attempt.patchOutcome,
+            attempt.verificationAvailability,
+            m(attempt.taskSuccess),
+            `${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`,
+            m(attempt.providerDurationMs),
+            attempt.tokenAvailability === "available" ? m(attempt.providerTokens) : "unavailable",
+            `proposed: ${attempt.proposedPatchPath ?? "none"}; applied: ${attempt.appliedPatchPath ?? "none"}; agent: ${attempt.agentArtifactDirectory ?? "none"}`
+        ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-edit-quality">
+    <h2>Edit Quality And Blast Radius (final evaluated patch)</h2>
+    ${table(["Case", "Treatment", "Expected-edit coverage", "Edit-scope precision", "Unexpected changed files", "Protected mutations", "Changed files", "Total churn", "Relative churn"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.expectedEditCoverage),
+        m(treatment.editScopePrecision),
+        m(treatment.unexpectedChangedFiles),
+        m(treatment.protectedMutations),
+        `${m(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}`,
+        m(treatment.totalChurn),
+        m(treatment.relativeChurn)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cost">
+    <h2>Duration And Token Evidence</h2>
+    ${table(["Case", "Treatment", "Final-attempt provider duration", "Final-attempt provider tokens", "Total provider duration (all attempts)", "Total provider tokens (all attempts)"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.finalAttemptProviderDurationMs),
+        m(treatment.finalAttemptProviderTokens),
+        treatment.totalProviderDurationMs ? m(treatment.totalProviderDurationMs) : "not applicable",
+        treatment.totalProviderTokens ? m(treatment.totalProviderTokens) : "not applicable"
+    ])))}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .flatMap((treatment) => [
+        ...treatment.repair.providerDurationMs.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider duration [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`),
+        ...treatment.repair.providerTokens.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider tokens [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`)
+    ])
+        .join("")}
+  </section>`);
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => [entry.caseId, treatment.treatmentId, String(error.attemptNumber), error.code, error.message])));
+    parts.push(`<section id="agent-success-rate-findings">
+    <h2>Agent Success Rate Warnings, Failures And Artifacts</h2>
+    ${report.warnings.length === 0 && errors.length === 0 ? "<p>No warnings or recorded errors.</p>" : table(["Source", "Case", "Treatment", "Attempt", "Code", "Message"], [...report.warnings.map((warning) => ["warning", "", "", "", warning.code, warning.message]), ...errors.map((error) => ["error", ...error])])}
+    ${table(["Artifact", "Reference", "Case", "Treatment"], report.artifacts.map((artifact) => [artifact.id, artifact.path ?? "[reference withheld]", artifact.caseId ?? "", artifact.variantId ?? ""]))}
+  </section>`);
+    return parts.join("\n\n  ");
+}