@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/experiments/plugins/agentSuccessRate/analysis.js

@@ -0,0 +1,425 @@
+import { AGENT_SUCCESS_MEAN_SOURCES, AGENT_SUCCESS_METRIC_IDS } from "./analysisTypes.js";
+import { AGENT_SUCCESS_RATE_EXECUTION_MODE, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+import { availableMetric, notApplicableMetric, ratioMetric, triMetric, unavailableMetric } from "./metrics.js";
+import { aggregateRepair, buildRepairComparison } from "./repairAnalysis.js";
+import { classifyAgentSuccessAttempt } from "./repairPolicy.js";
+export const AGENT_SUCCESS_RATE_LIMITATIONS = [
+    "Deterministic-fixture mode applies the same fixture patch to both treatment identities; the patch is not generated from any supplied context.",
+    "contextEffectEvaluated is false: raw-full-file versus context-pack differences are pipeline diagnostics, not performance evidence.",
+    "No coding agent runs, so no agent success rate, agent duration or provider token usage exists.",
+    "Verification counts are command/check level; individual test counts are not inferred from command output.",
+    "No weighted score, ranking, winner or significance claim is produced."
+];
+export const AGENT_SUCCESS_RATE_REAL_AGENT_LIMITATIONS = [
+    "Real-agent mode gives each treatment one initial provider attempt and, only when repairAttempts is configured, at most that many bounded repair attempts (three attempts in total); the provider generated its own patch from only the supplied treatment context.",
+    "Initial-attempt success and final success are separate measurements; a repaired success is never an initial-attempt success, and repair feedback is limited to fixed outcome categories.",
+    "Task success is derived from trusted checks on the actual changed repository state; agent prose is never evidence of success.",
+    "Matched-case comparisons are descriptive. A single run supports no causal, ranking, winner or statistical-significance claim.",
+    "Estimated context tokens are size estimates of the supplied context and are not provider-reported token usage.",
+    "Cases with an unavailable treatment verdict are excluded from matched metrics and reported separately.",
+    "Verification counts are command/check level; individual test counts are not inferred from command output.",
+    "No weighted score, ranking, winner or significance claim is produced."
+];
+const STATE_REASON = {
+    "infrastructure-failure": "execution infrastructure failed before the attempt could be evaluated.",
+    "baseline-invalid": "the benchmark baseline is not evaluable, so no implementation attempt was evaluated.",
+    "no-answer": "the provider produced no usable answer, so no implementation attempt was evaluated.",
+    "no-evidence": "required execution evidence is missing.",
+    "patch-failed": "the fixture patch was not applied, so post-edit verification did not run."
+};
+function and3(...values) {
+    if (values.includes(false))
+        return false;
+    if (values.includes("unknown"))
+        return "unknown";
+    return true;
+}
+function stateOf(evidence) {
+    if (evidence.availability === "infrastructure-failure")
+        return "infrastructure-failure";
+    if (evidence.baselineAssessment === null || evidence.availability === "baseline-invalid" || !evidence.baselineAssessment.evaluable)
+        return "baseline-invalid";
+    if (!evidence.patch.attempted || evidence.patch.outcome === null)
+        return evidence.realAgent ? "no-answer" : "no-evidence";
+    return evidence.patch.outcome === "success" ? "patched" : "patch-failed";
+}
+/** passed -> true, failed -> false, timeout/error/missing -> unknown. */
+function checkTri(result) {
+    if (result === undefined)
+        return "unknown";
+    if (result.status === "passed")
+        return true;
+    if (result.status === "failed")
+        return false;
+    return "unknown";
+}
+/** Any failed -> false; else any missing/indeterminate -> unknown; else true. */
+function allChecks(ids, results) {
+    return and3(...ids.map((id) => checkTri(results.get(id))));
+}
+function byId(results) {
+    return new Map((results ?? []).map((result) => [result.checkId, result]));
+}
+function duration(id, value) {
+    return value !== null && Number.isFinite(value) && value >= 0
+        ? availableMetric(id, value, "ms")
+        : unavailableMetric(id, "ms", "the duration was not measured.");
+}
+/**
+ * The single owner of per-case/per-treatment scientific values. Inputs are the validated task and normalized execution
+ * evidence only; agent prose and fixture notes are never consulted.
+ */
+export function analyzeAgentSuccessTreatment(task, evidence) {
+    const state = stateOf(evidence);
+    const gate = state === "patched" ? null : state === "patch-failed" && evidence.realAgent ? "the provider patch was not applied, so post-edit verification did not run." : STATE_REASON[state];
+    const m = {};
+    const un = (id, unit, reason) => unavailableMetric(id, unit, reason);
+    const post = evidence.postEditVerification;
+    const postTask = byId(post?.taskResults);
+    const postRegression = byId(post?.regressionResults);
+    const postAll = new Map([...postTask, ...postRegression]);
+    const baselineRegressionPassed = new Set((evidence.baselineVerification?.regressionResults ?? []).filter((r) => r.status === "passed").map((r) => r.checkId));
+    const postEvidenceMissing = state === "patched" && post === null ? "post-edit verification evidence is missing." : null;
+    // --- verification-derived tri-states (only meaningful once a patch was applied) ---
+    const taskIds = task.taskChecks.map((c) => c.id);
+    const regressionIds = task.regressionChecks.filter((c) => baselineRegressionPassed.has(c.id)).map((c) => c.id);
+    const patched = state === "patched" && post !== null;
+    const taskChecksTri = patched ? allChecks(taskIds, postTask) : "unknown";
+    const regressionTri = patched ? allChecks(regressionIds, postRegression) : "unknown";
+    const factState = (fact) => (patched ? allChecks(fact.verificationCheckIds, postAll) : "unknown");
+    const requiredFacts = task.behaviorFacts.filter((fact) => fact.required);
+    const optionalFacts = task.behaviorFacts.filter((fact) => !fact.required);
+    const requiredStates = requiredFacts.map(factState);
+    const optionalStates = optionalFacts.map(factState);
+    const requiredTri = and3(...requiredStates);
+    const integrity = evidence.protectedIntegrity.status;
+    const integrityTri = integrity === "intact" ? true : integrity === "mutated" ? false : "unknown";
+    const patchApplied = state === "patched";
+    const reasonFor = (fallback) => gate ?? postEvidenceMissing ?? fallback;
+    // --- task success ---
+    m.patchApplied = state === "patched" || state === "patch-failed" ? availableMetric("patchApplied", patchApplied, "boolean") : un("patchApplied", "boolean", gate ?? STATE_REASON["no-evidence"]);
+    if (state === "patch-failed") {
+        m.taskChecksPassed = un("taskChecksPassed", "boolean", gate);
+        m.regressionSafe = un("regressionSafe", "boolean", gate);
+        m.requiredFactsSatisfied = un("requiredFactsSatisfied", "boolean", gate);
+    }
+    else {
+        m.taskChecksPassed = triMetric("taskChecksPassed", taskChecksTri, reasonFor("a task check was missing or indeterminate."));
+        m.regressionSafe = triMetric("regressionSafe", regressionTri, reasonFor("a regression check was missing or indeterminate."));
+        m.requiredFactsSatisfied = triMetric("requiredFactsSatisfied", requiredTri, reasonFor("a required fact's check evidence was missing or indeterminate."));
+    }
+    m.protectedIntegrity =
+        state === "baseline-invalid" || state === "infrastructure-failure" || state === "no-evidence" || state === "no-answer"
+            ? un("protectedIntegrity", "boolean", gate)
+            : triMetric("protectedIntegrity", integrityTri, "protected-file integrity could not be proven.");
+    const resolved = state === "patched" ? and3(true, taskChecksTri, requiredTri) : state === "patch-failed" ? false : "unknown";
+    const success = state === "patched" ? and3(resolved, regressionTri, integrityTri) : state === "patch-failed" ? false : "unknown";
+    m.taskResolved = triMetric("taskResolved", resolved, gate ?? "a task check or required fact could not be determined.");
+    m.taskSuccess = triMetric("taskSuccess", success, gate ?? "a component of task success could not be determined.");
+    // --- behavior facts ---
+    const determinate = (states) => states.every((s) => s !== "unknown");
+    const count = (states) => states.filter((s) => s === true).length;
+    m.requiredFactsTotal = availableMetric("requiredFactsTotal", requiredFacts.length, "count");
+    m.optionalFactsTotal = availableMetric("optionalFactsTotal", optionalFacts.length, "count");
+    m.requiredFactsSatisfiedCount = determinate(requiredStates) && patched ? availableMetric("requiredFactsSatisfiedCount", count(requiredStates), "count") : un("requiredFactsSatisfiedCount", "count", reasonFor("a required fact could not be determined."));
+    m.optionalFactsSatisfiedCount = determinate(optionalStates) && patched ? availableMetric("optionalFactsSatisfiedCount", count(optionalStates), "count") : un("optionalFactsSatisfiedCount", "count", reasonFor("an optional fact could not be determined."));
+    m.requiredFactCoverage =
+        requiredFacts.length === 0
+            ? notApplicableMetric("requiredFactCoverage", "ratio", "the task defines no required facts.")
+            : determinate(requiredStates) && patched
+                ? ratioMetric("requiredFactCoverage", count(requiredStates), requiredFacts.length, "no required facts.")
+                : un("requiredFactCoverage", "ratio", reasonFor("a required fact could not be determined."));
+    const allStates = [...requiredStates, ...optionalStates];
+    m.factCoverage =
+        task.behaviorFacts.length === 0
+            ? notApplicableMetric("factCoverage", "ratio", "the task defines no behavior facts.")
+            : determinate(allStates) && patched
+                ? ratioMetric("factCoverage", count(allStates), task.behaviorFacts.length, "no facts.")
+                : un("factCoverage", "ratio", reasonFor("a behavior fact could not be determined."));
+    // --- check-level counts ---
+    const taskComplete = patched && taskIds.every((id) => postTask.has(id) && postTask.get(id).status !== "timeout" && postTask.get(id).status !== "error");
+    const regressionAll = task.regressionChecks.map((c) => c.id);
+    const regressionComplete = patched && regressionAll.every((id) => postRegression.has(id) && postRegression.get(id).status !== "timeout" && postRegression.get(id).status !== "error");
+    const passedOf = (ids, results) => ids.filter((id) => results.get(id)?.status === "passed").length;
+    const checkReason = (kind) => reasonFor(`a ${kind} check was missing, timed out or errored; complete check evidence is required.`);
+    if (taskComplete) {
+        m.taskCheckPassedCount = availableMetric("taskCheckPassedCount", passedOf(taskIds, postTask), "count");
+        m.taskCheckTotalCount = availableMetric("taskCheckTotalCount", taskIds.length, "count");
+        m.taskCheckPassRate = ratioMetric("taskCheckPassRate", passedOf(taskIds, postTask), taskIds.length, "the task defines no task checks.");
+    }
+    else {
+        m.taskCheckPassedCount = un("taskCheckPassedCount", "count", checkReason("task"));
+        m.taskCheckTotalCount = un("taskCheckTotalCount", "count", checkReason("task"));
+        m.taskCheckPassRate = un("taskCheckPassRate", "ratio", checkReason("task"));
+    }
+    if (regressionComplete) {
+        m.regressionCheckPassedCount = availableMetric("regressionCheckPassedCount", passedOf(regressionAll, postRegression), "count");
+        m.regressionCheckTotalCount = availableMetric("regressionCheckTotalCount", regressionAll.length, "count");
+        m.regressionCheckPassRate = ratioMetric("regressionCheckPassRate", passedOf(regressionAll, postRegression), regressionAll.length, "the task defines no regression checks.");
+        m.regressionFailureCount = availableMetric("regressionFailureCount", regressionIds.filter((id) => postRegression.get(id)?.status === "failed").length, "count");
+    }
+    else {
+        for (const id of ["regressionCheckPassedCount", "regressionCheckTotalCount", "regressionFailureCount"])
+            m[id] = un(id, "count", checkReason("regression"));
+        m.regressionCheckPassRate = un("regressionCheckPassRate", "ratio", checkReason("regression"));
+    }
+    // --- edit quality and blast radius (ChangeSet evidence only) ---
+    const change = patched ? evidence.change : null;
+    const changeReason = reasonFor("change evidence was not captured.");
+    const expected = new Set(task.expectedEditFiles);
+    const allowed = new Set(task.allowedEditFiles);
+    m.expectedEditFileCount = availableMetric("expectedEditFileCount", expected.size, "count");
+    if (change === null) {
+        for (const id of ["expectedEditFilesChangedCount", "allowedEditChangedFileCount", "unexpectedChangedFileCount", "changedFileCount", "addedFileCount", "modifiedFileCount", "deletedFileCount", "linesAdded", "linesDeleted", "totalChurn"]) {
+            m[id] = un(id, id.startsWith("lines") || id === "totalChurn" ? "lines" : "count", changeReason);
+        }
+        m.expectedEditCoverage = un("expectedEditCoverage", "ratio", changeReason);
+        m.editScopePrecision = un("editScopePrecision", "ratio", changeReason);
+        m.relativeChurn = un("relativeChurn", "ratio", changeReason);
+    }
+    else {
+        const paths = change.changedFiles.map((file) => file.relativePath);
+        const expectedChanged = paths.filter((p) => expected.has(p)).length;
+        const allowedChanged = paths.filter((p) => allowed.has(p)).length;
+        const statusCount = (status) => change.changedFiles.filter((file) => file.status === status).length;
+        m.expectedEditFilesChangedCount = availableMetric("expectedEditFilesChangedCount", expectedChanged, "count");
+        m.expectedEditCoverage = ratioMetric("expectedEditCoverage", expectedChanged, expected.size, "the task expects no edited files.");
+        m.allowedEditChangedFileCount = availableMetric("allowedEditChangedFileCount", allowedChanged, "count");
+        m.unexpectedChangedFileCount = availableMetric("unexpectedChangedFileCount", paths.length - allowedChanged, "count");
+        m.editScopePrecision = ratioMetric("editScopePrecision", allowedChanged, paths.length, "no files were changed.");
+        m.changedFileCount = availableMetric("changedFileCount", paths.length, "count");
+        m.addedFileCount = availableMetric("addedFileCount", statusCount("added"), "count");
+        m.modifiedFileCount = availableMetric("modifiedFileCount", statusCount("modified"), "count");
+        m.deletedFileCount = availableMetric("deletedFileCount", statusCount("deleted"), "count");
+        const hasBinary = change.changedFiles.some((file) => file.additions === null || file.deletions === null);
+        if (hasBinary) {
+            const reason = "a changed file has no line counts (binary); line metrics are not estimated.";
+            m.linesAdded = un("linesAdded", "lines", reason);
+            m.linesDeleted = un("linesDeleted", "lines", reason);
+            m.totalChurn = un("totalChurn", "lines", reason);
+            m.relativeChurn = un("relativeChurn", "ratio", reason);
+        }
+        else {
+            const added = change.changedFiles.reduce((sum, file) => sum + (file.additions ?? 0), 0);
+            const deleted = change.changedFiles.reduce((sum, file) => sum + (file.deletions ?? 0), 0);
+            m.linesAdded = availableMetric("linesAdded", added, "lines");
+            m.linesDeleted = availableMetric("linesDeleted", deleted, "lines");
+            m.totalChurn = availableMetric("totalChurn", added + deleted, "lines");
+            const baseLines = evidence.baselineTextLineCount;
+            m.relativeChurn =
+                baseLines === null
+                    ? un("relativeChurn", "ratio", "the baseline text-line count could not be determined.")
+                    : ratioMetric("relativeChurn", added + deleted, baseLines, "the baseline has no text lines.");
+        }
+    }
+    m.baselineTextLineCount =
+        evidence.baselineTextLineCount === null
+            ? un("baselineTextLineCount", "lines", "the baseline text-line count could not be determined.")
+            : availableMetric("baselineTextLineCount", evidence.baselineTextLineCount, "lines");
+    // --- protected edits: actual mutation vs. attempted ---
+    const mutationEvaluable = (state === "patched" || state === "patch-failed") && integrity !== "unproven";
+    m.protectedMutationCount = mutationEvaluable
+        ? availableMetric("protectedMutationCount", evidence.protectedIntegrity.mutatedPaths.length, "count")
+        : un("protectedMutationCount", "count", gate ?? "protected-file integrity could not be proven.");
+    m.attemptedProtectedEditCount = evidence.patch.attempted
+        ? availableMetric("attemptedProtectedEditCount", evidence.patch.attemptedProtectedPaths.length, "count")
+        : un("attemptedProtectedEditCount", "count", "no patch was attempted.");
+    // --- time and tokens ---
+    const real = evidence.realAgent;
+    if (!real) {
+        m.agentDurationMs = un("agentDurationMs", "ms", "deterministic-fixture mode runs no agent.");
+        m.agentTotalTokens = un("agentTotalTokens", "tokens", "deterministic-fixture mode has no provider token usage.");
+    }
+    else {
+        m.agentDurationMs = real.providerInvoked ? duration("agentDurationMs", evidence.timing.agentDurationMs) : un("agentDurationMs", "ms", "no provider attempt ran.");
+        const usage = evidence.agentTokenUsage;
+        m.agentTotalTokens =
+            usage !== null && usage.totalTokens !== null
+                ? availableMetric("agentTotalTokens", usage.totalTokens, "tokens")
+                : un("agentTotalTokens", "tokens", usage === null ? "no provider attempt ran." : `the provider reported no total token usage (source: ${usage.source}, reliability: ${usage.reliability}).`);
+    }
+    m.baselineVerificationDurationMs = duration("baselineVerificationDurationMs", evidence.timing.baselineVerificationDurationMs);
+    m.patchPipelineDurationMs = duration("patchPipelineDurationMs", evidence.timing.patchPipelineDurationMs);
+    m.postEditVerificationDurationMs = duration("postEditVerificationDurationMs", evidence.timing.postEditVerificationDurationMs);
+    m.evaluationDurationMs = duration("evaluationDurationMs", evidence.timing.evaluationDurationMs);
+    const metrics = {};
+    for (const id of AGENT_SUCCESS_METRIC_IDS) {
+        const metric = m[id];
+        if (!metric)
+            throw new Error(`Agent success analysis did not produce metric ${id}.`);
+        metrics[id] = metric;
+    }
+    const repair = evidence.attempts && evidence.attempts.length > 0 ? buildRepairAnalysis(task, evidence.attempts) : undefined;
+    return { treatmentId: evidence.treatmentId, caseId: task.id, executionStatus: evidence.status, evidenceAvailability: evidence.availability, metrics, ...(repair ? { repair } : {}) };
+}
+function retag(metric, id) {
+    return { ...metric, id };
+}
+/** Sum of a per-attempt quantity over the provider attempts that ran. Unavailable if any invoked attempt was not measured. */
+function sumOverInvoked(id, unit, noun, samples) {
+    const invoked = samples.filter((sample) => sample.invoked);
+    if (invoked.length === 0)
+        return unavailableMetric(id, unit, "no provider attempt ran.");
+    const missing = invoked.filter((sample) => sample.metric.availability !== "available");
+    if (missing.length > 0)
+        return unavailableMetric(id, unit, `${missing.length} of ${invoked.length} provider attempt(s) have no ${noun}; a complete total is not available.`);
+    return availableMetric(id, invoked.reduce((total, sample) => total + sample.metric.value, 0), unit);
+}
+/** Per-attempt analysis plus the initial/final separation. Attempt order is evidence order (contiguous, one-based). */
+function buildRepairAnalysis(task, attempts) {
+    const analyzed = attempts.map((attempt) => {
+        const analysis = analyzeAgentSuccessTreatment(task, attempt.evidence);
+        const classification = classifyAgentSuccessAttempt(attempt.evidence, analysis.metrics);
+        const entry = {
+            attemptNumber: attempt.attemptNumber,
+            providerStatus: attempt.providerStatus,
+            failureCategory: classification.category,
+            repairEligible: classification.repairEligible,
+            executionStatus: analysis.executionStatus,
+            evidenceAvailability: analysis.evidenceAvailability,
+            metrics: analysis.metrics
+        };
+        return { entry, invoked: attempt.evidence.realAgent?.providerInvoked === true };
+    });
+    const first = analyzed[0];
+    const last = analyzed[analyzed.length - 1];
+    const initial = retag(first.entry.metrics.taskSuccess, "initialAttemptTaskSuccess");
+    const final = retag(last.entry.metrics.taskSuccess, "finalTaskSuccess");
+    const repairAttemptCount = analyzed.length - 1;
+    let repairSucceeded;
+    if (repairAttemptCount === 0) {
+        repairSucceeded = notApplicableMetric("repairSucceeded", "boolean", "no repair attempt was executed.");
+    }
+    else if (initial.availability !== "available" || final.availability !== "available") {
+        repairSucceeded = unavailableMetric("repairSucceeded", "boolean", "the initial or final attempt has no determinate task-success verdict.");
+    }
+    else {
+        repairSucceeded = availableMetric("repairSucceeded", initial.value === false && final.value === true, "boolean");
+    }
+    const duration = (item) => item.entry.metrics.agentDurationMs;
+    const tokens = (item) => item.entry.metrics.agentTotalTokens;
+    const firstOr = (item, pick, id, unit) => item.invoked ? retag(pick(item), id) : unavailableMetric(id, unit, "no provider attempt ran.");
+    const evaluationDurations = analyzed.map((item) => item.entry.metrics.evaluationDurationMs);
+    const missingEvaluation = evaluationDurations.filter((metric) => metric.availability !== "available").length;
+    const invokedItems = analyzed.filter((item) => item.invoked);
+    return {
+        attemptCount: analyzed.length,
+        repairAttemptCount,
+        initialAttemptRepairEligible: first.entry.repairEligible,
+        initialAttemptTaskSuccess: initial,
+        finalTaskSuccess: final,
+        repairSucceeded,
+        firstAttemptProviderDurationMs: firstOr(first, duration, "firstAttemptProviderDurationMs", "ms"),
+        finalAttemptProviderDurationMs: firstOr(last, duration, "finalAttemptProviderDurationMs", "ms"),
+        totalProviderDurationMs: sumOverInvoked("totalProviderDurationMs", "ms", "duration measurement", analyzed.map((item) => ({ invoked: item.invoked, metric: duration(item) }))),
+        firstAttemptProviderTokens: firstOr(first, tokens, "firstAttemptProviderTokens", "tokens"),
+        finalAttemptProviderTokens: firstOr(last, tokens, "finalAttemptProviderTokens", "tokens"),
+        totalProviderTokens: sumOverInvoked("totalProviderTokens", "tokens", "provider-reported total token usage", analyzed.map((item) => ({ invoked: item.invoked, metric: tokens(item) }))),
+        totalEvaluationDurationMs: missingEvaluation > 0
+            ? unavailableMetric("totalEvaluationDurationMs", "ms", `${missingEvaluation} of ${analyzed.length} attempt(s) have no evaluation duration; a complete total is not available.`)
+            : availableMetric("totalEvaluationDurationMs", evaluationDurations.reduce((total, metric) => total + metric.value, 0), "ms"),
+        providerAttemptCount: invokedItems.length,
+        providerDurationMeasuredAttemptCount: invokedItems.filter((item) => duration(item).availability === "available").length,
+        providerTokenMeasuredAttemptCount: invokedItems.filter((item) => tokens(item).availability === "available").length,
+        attempts: analyzed.map((item) => item.entry)
+    };
+}
+function mean(id, caseIds, treatment, allTreatments) {
+    const sourceMetricId = AGENT_SUCCESS_MEAN_SOURCES[id];
+    const matched = caseIds.filter((caseId) => allTreatments.every((series) => series.find((entry) => entry.caseId === caseId)?.metrics[sourceMetricId].availability === "available"));
+    const unit = treatment[0]?.metrics[sourceMetricId].unit ?? "count";
+    if (matched.length === 0) {
+        return { sourceMetricId, matchedCaseIds: [], metric: unavailableMetric(id, unit, "no case has available evidence for this quantity in every treatment.") };
+    }
+    const values = matched.map((caseId) => treatment.find((entry) => entry.caseId === caseId).metrics[sourceMetricId].value);
+    return { sourceMetricId, matchedCaseIds: matched, metric: availableMetric(id, values.reduce((sum, value) => sum + value, 0) / values.length, unit) };
+}
+/**
+ * Per-case analysis plus unweighted aggregates. Means are taken over matched cases only (available for every
+ * treatment); unavailable values are never converted to zero. No composite, ranking or winner is produced.
+ */
+export function analyzeAgentSuccessRate(tasks, caseEvidence, executionMode = AGENT_SUCCESS_RATE_EXECUTION_MODE) {
+    if (tasks.length !== caseEvidence.length || tasks.some((task, index) => caseEvidence[index]?.caseId !== task.id)) {
+        throw new Error("Agent success analysis requires case evidence in the same order as the selected tasks.");
+    }
+    const cases = tasks.map((task, index) => {
+        const evidence = caseEvidence[index];
+        if (evidence.treatments.length !== AGENT_SUCCESS_RATE_TREATMENT_IDS.length || evidence.treatments.some((t, i) => t.treatmentId !== AGENT_SUCCESS_RATE_TREATMENT_IDS[i])) {
+            throw new Error(`Agent success analysis requires both treatments in fixed order for case ${task.id}.`);
+        }
+        return { caseId: task.id, benchmarkProject: task.benchmarkProject, treatments: evidence.treatments.map((t) => analyzeAgentSuccessTreatment(task, t)) };
+    });
+    const series = AGENT_SUCCESS_RATE_TREATMENT_IDS.map((_, treatmentIndex) => cases.map((c) => c.treatments[treatmentIndex]));
+    const caseIds = cases.map((c) => c.caseId);
+    const aggregates = AGENT_SUCCESS_RATE_TREATMENT_IDS.map((treatmentId, treatmentIndex) => {
+        const own = series[treatmentIndex];
+        const evaluable = own.filter((entry) => entry.metrics.taskSuccess.availability === "available");
+        const successful = evaluable.filter((entry) => entry.metrics.taskSuccess.value === true);
+        const rate = (id) => evaluable.length === 0 ? unavailableMetric(id, "ratio", "no case has a determinate task-success verdict for this treatment.") : availableMetric(id, successful.length / evaluable.length, "ratio");
+        const means = {};
+        for (const meanId of Object.keys(AGENT_SUCCESS_MEAN_SOURCES))
+            means[meanId] = mean(meanId, caseIds, own, series);
+        const repair = aggregateRepair(own);
+        return {
+            treatmentId,
+            evaluableCaseCount: evaluable.length,
+            successfulCaseCount: successful.length,
+            taskSuccessRate: rate("taskSuccessRate"),
+            // With repair evidence the initial-attempt rate comes from attempt 1; otherwise one attempt makes it equal the final rate.
+            initialAttemptSuccessRate: repair ? repair.initialAttemptSuccessRate : rate("initialAttemptSuccessRate"),
+            agentTokenMeasurementsAvailable: own.filter((entry) => entry.metrics.agentTotalTokens.availability === "available").length,
+            agentTokenMeasurementsUnavailable: own.filter((entry) => entry.metrics.agentTotalTokens.availability !== "available").length,
+            means,
+            ...(repair ? { repair } : {})
+        };
+    });
+    if (executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE) {
+        const comparison = buildComparison(cases);
+        const hasRepairEvidence = cases.some((entry) => entry.treatments.some((treatment) => treatment.repair !== undefined));
+        return {
+            executionMode,
+            contextEffectEvaluated: comparison.matchedCaseIds.length > 0,
+            treatmentOrder: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+            caseCount: cases.length,
+            cases,
+            aggregates,
+            limitations: [...AGENT_SUCCESS_RATE_REAL_AGENT_LIMITATIONS],
+            comparison,
+            ...(hasRepairEvidence ? { repairComparison: buildRepairComparison(cases, comparison.pairedOutcomes) } : {})
+        };
+    }
+    return {
+        executionMode: AGENT_SUCCESS_RATE_EXECUTION_MODE,
+        contextEffectEvaluated: false,
+        treatmentOrder: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+        caseCount: cases.length,
+        cases,
+        aggregates,
+        limitations: [...AGENT_SUCCESS_RATE_LIMITATIONS]
+    };
+}
+/** Descriptive matched-case accounting: only cases with a determinate verdict for every treatment are matched. */
+function buildComparison(cases) {
+    const matchedCaseIds = [];
+    const incompleteCases = [];
+    const pairedOutcomes = { bothSucceeded: 0, onlyRawFullFileSucceeded: 0, onlyContextPackSucceeded: 0, neitherSucceeded: 0 };
+    for (const entry of cases) {
+        const unavailableTreatmentIds = entry.treatments.filter((t) => t.metrics.taskSuccess.availability !== "available").map((t) => t.treatmentId);
+        if (unavailableTreatmentIds.length > 0) {
+            incompleteCases.push({ caseId: entry.caseId, unavailableTreatmentIds });
+            continue;
+        }
+        matchedCaseIds.push(entry.caseId);
+        const raw = entry.treatments[0].metrics.taskSuccess.value === true;
+        const pack = entry.treatments[1].metrics.taskSuccess.value === true;
+        if (raw && pack)
+            pairedOutcomes.bothSucceeded += 1;
+        else if (raw)
+            pairedOutcomes.onlyRawFullFileSucceeded += 1;
+        else if (pack)
+            pairedOutcomes.onlyContextPackSucceeded += 1;
+        else
+            pairedOutcomes.neitherSucceeded += 1;
+    }
+    return { basis: "matched-evaluable-cases", matchedCaseIds, incompleteCases, pairedOutcomes };
+}

package/dist/src/experiments/plugins/agentSuccessRate/analysisArtifact.js

@@ -0,0 +1,155 @@
+import { AGENT_SUCCESS_METRIC_IDS } from "./analysisTypes.js";
+import { AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+import { validateAgentSuccessMetric } from "./metrics.js";
+export const AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE = "agent-success-rate-analysis.json";
+export const AGENT_SUCCESS_RATE_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-rate-analysis-v1";
+export const AGENT_SUCCESS_RATE_METHODOLOGY = {
+    taskSuccess: "resolved-and-regression-safe-and-protected-intact",
+    factSatisfaction: "all-referenced-post-edit-checks-passed",
+    verificationGranularity: "command-check-level",
+    aggregation: "matched-case-unweighted-mean",
+    treatmentComparison: "pipeline-diagnostic-only",
+    relativeChurnDenominator: "baseline-text-line-count"
+};
+export function buildAgentSuccessRateAnalysisArtifact(args) {
+    return {
+        schemaVersion: AGENT_SUCCESS_RATE_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: {
+            ...AGENT_SUCCESS_RATE_METHODOLOGY,
+            ...(args.analysis.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE ? { treatmentComparison: "descriptive-matched-case" } : {})
+        },
+        analysis: structuredClone(args.analysis)
+    };
+}
+/** Structural validation of an in-memory analysis artifact. Returns problems; empty means valid. */
+export function validateAgentSuccessRateAnalysisArtifact(artifact) {
+    const problems = [];
+    if (artifact.schemaVersion !== AGENT_SUCCESS_RATE_ANALYSIS_SCHEMA_VERSION)
+        problems.push("analysis artifact schemaVersion is not recognized.");
+    if (artifact.pluginId !== AGENT_SUCCESS_RATE_PLUGIN_ID)
+        problems.push("analysis artifact pluginId does not match.");
+    const { analysis } = artifact;
+    if (analysis.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE) {
+        if (!analysis.comparison)
+            problems.push("real-agent analysis is missing the matched-case comparison.");
+        else if (analysis.contextEffectEvaluated !== analysis.comparison.matchedCaseIds.length > 0)
+            problems.push("analysis contextEffectEvaluated disagrees with the matched cases.");
+        if (artifact.methodology.treatmentComparison !== "descriptive-matched-case")
+            problems.push("real-agent analysis must use the descriptive matched-case methodology.");
+    }
+    else {
+        if (analysis.contextEffectEvaluated !== false)
+            problems.push("analysis must state contextEffectEvaluated=false.");
+        if (analysis.comparison !== undefined)
+            problems.push("deterministic-fixture analysis must not carry a comparison.");
+    }
+    if (analysis.treatmentOrder.join(",") !== AGENT_SUCCESS_RATE_TREATMENT_IDS.join(","))
+        problems.push("analysis treatmentOrder is not the fixed order.");
+    for (const entry of analysis.cases) {
+        if (entry.treatments.map((t) => t.treatmentId).join(",") !== AGENT_SUCCESS_RATE_TREATMENT_IDS.join(","))
+            problems.push(`analysis case ${entry.caseId} does not hold both treatments in fixed order.`);
+        for (const treatment of entry.treatments) {
+            for (const id of AGENT_SUCCESS_METRIC_IDS) {
+                const metric = treatment.metrics[id];
+                if (!metric)
+                    problems.push(`analysis case ${entry.caseId}/${treatment.treatmentId} is missing metric ${id}.`);
+                else
+                    problems.push(...validateAgentSuccessMetric(metric));
+            }
+            if (analysis.executionMode !== AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE && treatment.repair !== undefined) {
+                problems.push(`analysis case ${entry.caseId}/${treatment.treatmentId} carries repair analysis in deterministic-fixture mode.`);
+            }
+            if (treatment.repair) {
+                const repair = treatment.repair;
+                const label = `analysis case ${entry.caseId}/${treatment.treatmentId}`;
+                for (const metric of [
+                    repair.initialAttemptTaskSuccess,
+                    repair.finalTaskSuccess,
+                    repair.repairSucceeded,
+                    repair.firstAttemptProviderDurationMs,
+                    repair.finalAttemptProviderDurationMs,
+                    repair.totalProviderDurationMs,
+                    repair.firstAttemptProviderTokens,
+                    repair.finalAttemptProviderTokens,
+                    repair.totalProviderTokens,
+                    repair.totalEvaluationDurationMs
+                ]) {
+                    problems.push(...validateAgentSuccessMetric(metric));
+                }
+                if (repair.attempts.length !== repair.attemptCount || repair.repairAttemptCount !== repair.attemptCount - 1)
+                    problems.push(`${label} attempt counts are inconsistent.`);
+                repair.attempts.forEach((attempt, index) => {
+                    if (attempt.attemptNumber !== index + 1)
+                        problems.push(`${label} attempt analysis numbering is not contiguous from 1.`);
+                    for (const id of AGENT_SUCCESS_METRIC_IDS) {
+                        const metric = attempt.metrics[id];
+                        if (!metric)
+                            problems.push(`${label} attempt ${attempt.attemptNumber} is missing metric ${id}.`);
+                        else
+                            problems.push(...validateAgentSuccessMetric(metric));
+                    }
+                });
+                const final = repair.attempts[repair.attempts.length - 1];
+                if (final && JSON.stringify(final.metrics.taskSuccess.value) !== JSON.stringify(treatment.metrics.taskSuccess.value))
+                    problems.push(`${label} final attempt analysis disagrees with the final metrics.`);
+            }
+        }
+    }
+    for (const aggregate of analysis.aggregates) {
+        problems.push(...validateAgentSuccessMetric(aggregate.taskSuccessRate), ...validateAgentSuccessMetric(aggregate.initialAttemptSuccessRate));
+        if (aggregate.repair) {
+            const repair = aggregate.repair;
+            problems.push(...validateAgentSuccessMetric(repair.initialAttemptSuccessRate), ...validateAgentSuccessMetric(repair.finalTaskSuccessRate), ...validateAgentSuccessMetric(repair.repairSuccessRate), ...validateAgentSuccessMetric(repair.meanAttemptsPerEvaluableCase));
+            for (const total of [...repair.providerDurationMs, ...repair.providerTokens])
+                problems.push(...validateAgentSuccessMetric(total.total));
+        }
+        for (const mean of Object.values(aggregate.means))
+            problems.push(...validateAgentSuccessMetric(mean.metric));
+    }
+    return problems;
+}
+/** Execution and analysis must describe the same run, cases and treatments. */
+export function validateAgentSuccessRateArtifactFamily(execution, analysis) {
+    const problems = [];
+    if (execution.runId !== analysis.runId)
+        problems.push("execution and analysis runId differ.");
+    if (execution.pluginSchemaVersion !== analysis.pluginSchemaVersion)
+        problems.push("execution and analysis pluginSchemaVersion differ.");
+    if (execution.executionMode !== analysis.analysis.executionMode)
+        problems.push("execution and analysis executionMode differ.");
+    if (execution.contextEffectEvaluated !== analysis.analysis.contextEffectEvaluated)
+        problems.push("execution and analysis contextEffectEvaluated differ.");
+    if (execution.cases.length !== analysis.analysis.cases.length)
+        problems.push("execution and analysis case counts differ.");
+    execution.cases.forEach((entry, index) => {
+        const counterpart = analysis.analysis.cases[index];
+        if (!counterpart || counterpart.caseId !== entry.caseId) {
+            problems.push(`execution and analysis case identity differs at index ${index}.`);
+            return;
+        }
+        if (entry.treatments.map((t) => t.treatmentId).join(",") !== counterpart.treatments.map((t) => t.treatmentId).join(",")) {
+            problems.push(`execution and analysis treatment identity differs for case ${entry.caseId}.`);
+        }
+        entry.treatments.forEach((treatment, treatmentIndex) => {
+            const executedAttempts = treatment.attempts?.length ?? null;
+            const analyzedAttempts = counterpart.treatments[treatmentIndex]?.repair?.attempts.length ?? null;
+            if (executedAttempts !== analyzedAttempts)
+                problems.push(`execution and analysis attempt counts differ for case ${entry.caseId}/${treatment.treatmentId}.`);
+            treatment.attempts?.forEach((attempt, attemptIndex) => {
+                const analyzed = counterpart.treatments[treatmentIndex]?.repair?.attempts[attemptIndex];
+                if (analyzed && analyzed.attemptNumber !== attempt.attemptNumber)
+                    problems.push(`execution and analysis attempt identity differs for case ${entry.caseId}/${treatment.treatmentId}.`);
+                const verdict = analyzed?.metrics.taskSuccess;
+                const analyzedSuccess = verdict && verdict.availability === "available" ? verdict.value : null;
+                if (analyzed && analyzedSuccess !== attempt.taskSuccess)
+                    problems.push(`execution and analysis task success differ for case ${entry.caseId}/${treatment.treatmentId} attempt ${attempt.attemptNumber}.`);
+            });
+        });
+    });
+    return problems;
+}

package/dist/src/experiments/plugins/agentSuccessRate/analysisTypes.js

@@ -0,0 +1,65 @@
+/** Stable metric IDs, in the order they are written to artifacts. */
+export const AGENT_SUCCESS_METRIC_IDS = [
+    // task success components
+    "patchApplied",
+    "taskChecksPassed",
+    "regressionSafe",
+    "requiredFactsSatisfied",
+    "protectedIntegrity",
+    "taskResolved",
+    "taskSuccess",
+    // behavior facts
+    "requiredFactsTotal",
+    "requiredFactsSatisfiedCount",
+    "optionalFactsTotal",
+    "optionalFactsSatisfiedCount",
+    "requiredFactCoverage",
+    "factCoverage",
+    // verification checks (command/check level only)
+    "taskCheckPassedCount",
+    "taskCheckTotalCount",
+    "taskCheckPassRate",
+    "regressionCheckPassedCount",
+    "regressionCheckTotalCount",
+    "regressionCheckPassRate",
+    "regressionFailureCount",
+    // edit quality
+    "expectedEditFileCount",
+    "expectedEditFilesChangedCount",
+    "expectedEditCoverage",
+    "allowedEditChangedFileCount",
+    "unexpectedChangedFileCount",
+    "editScopePrecision",
+    "protectedMutationCount",
+    "attemptedProtectedEditCount",
+    // blast radius
+    "changedFileCount",
+    "addedFileCount",
+    "modifiedFileCount",
+    "deletedFileCount",
+    "linesAdded",
+    "linesDeleted",
+    "totalChurn",
+    "baselineTextLineCount",
+    "relativeChurn",
+    // time and tokens
+    "agentDurationMs",
+    "agentTotalTokens",
+    "baselineVerificationDurationMs",
+    "patchPipelineDurationMs",
+    "postEditVerificationDurationMs",
+    "evaluationDurationMs"
+];
+/** Aggregate mean ID -> the per-case metric it averages. */
+export const AGENT_SUCCESS_MEAN_SOURCES = {
+    meanTaskCheckPassRate: "taskCheckPassRate",
+    meanRegressionCheckPassRate: "regressionCheckPassRate",
+    meanRequiredFactCoverage: "requiredFactCoverage",
+    meanExpectedEditCoverage: "expectedEditCoverage",
+    meanEditScopePrecision: "editScopePrecision",
+    meanUnexpectedChangedFileCount: "unexpectedChangedFileCount",
+    meanChangedFileCount: "changedFileCount",
+    meanTotalChurn: "totalChurn",
+    meanRelativeChurn: "relativeChurn",
+    meanEvaluationDurationMs: "evaluationDurationMs"
+};