@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/experiments/plugins/agentSuccessRate/metrics.js

@@ -0,0 +1,76 @@
+const PREFIX = "agent-success-rate";
+/** Typed metric constructors. An absent measurement is never represented as zero. */
+export function availableMetric(id, value, unit) {
+    if (typeof value === "number" && !Number.isFinite(value)) {
+        return unavailableMetric(id, unit, "the computed value was not finite.");
+    }
+    return { id, availability: "available", value, unit, reason: null };
+}
+export function unavailableMetric(id, unit, reason) {
+    return { id, availability: "unavailable", value: null, unit, reason: reason.trim() || "evidence is unavailable." };
+}
+export function notApplicableMetric(id, unit, reason) {
+    return { id, availability: "not-applicable", value: null, unit, reason: reason.trim() || "not applicable." };
+}
+export function triMetric(id, value, unknownReason) {
+    return value === "unknown" ? unavailableMetric(id, "boolean", unknownReason) : availableMetric(id, value, "boolean");
+}
+/** ratio = numerator / denominator, or not-applicable when the denominator is zero. */
+export function ratioMetric(id, numerator, denominator, zeroReason) {
+    return denominator === 0 ? notApplicableMetric(id, "ratio", zeroReason) : availableMetric(id, numerator / denominator, "ratio");
+}
+/** Structural check of the metric availability contract. Returns problems; empty means valid. */
+export function validateAgentSuccessMetric(metric) {
+    const problems = [];
+    if (metric.availability === "available") {
+        if (metric.value === null)
+            problems.push(`${metric.id}: available metric has a null value.`);
+        if (typeof metric.value === "number" && !Number.isFinite(metric.value))
+            problems.push(`${metric.id}: available metric is not finite.`);
+        if (metric.reason !== null)
+            problems.push(`${metric.id}: available metric must not carry a reason.`);
+    }
+    else {
+        if (metric.value !== null)
+            problems.push(`${metric.id}: ${metric.availability} metric must have a null value.`);
+        if (typeof metric.reason !== "string" || metric.reason.trim() === "")
+            problems.push(`${metric.id}: ${metric.availability} metric needs a reason.`);
+    }
+    return problems;
+}
+/** Flattening to the generic ExperimentMetric happens only here, for generic run/outcome compatibility. */
+export function toExperimentMetric(metric, scope) {
+    return {
+        id: `${PREFIX}.${metric.id}`,
+        name: scope.name ?? metric.id,
+        value: metric.value,
+        unit: metric.unit,
+        description: metric.availability === "available" ? "available" : `${metric.availability}: ${metric.reason ?? ""}`,
+        ...(scope.variantId === undefined ? {} : { variantId: scope.variantId }),
+        ...(scope.caseId === undefined ? {} : { caseId: scope.caseId })
+    };
+}
+export function toAgentSuccessOutcomeMetrics(analysis) {
+    const scope = { caseId: analysis.caseId, variantId: analysis.treatmentId };
+    const metrics = Object.keys(analysis.metrics).map((id) => toExperimentMetric(analysis.metrics[id], scope));
+    const repair = analysis.repair;
+    if (repair) {
+        // Initial-attempt and final outcomes are separate measurements; attempt counts are plain counts.
+        metrics.push(toExperimentMetric(repair.initialAttemptTaskSuccess, scope), toExperimentMetric(repair.finalTaskSuccess, scope), toExperimentMetric(repair.repairSucceeded, scope), toExperimentMetric(availableMetric("attemptCount", repair.attemptCount, "count"), scope), toExperimentMetric(availableMetric("repairAttemptCount", repair.repairAttemptCount, "count"), scope), toExperimentMetric(repair.totalProviderDurationMs, scope), toExperimentMetric(repair.totalProviderTokens, scope), toExperimentMetric(repair.totalEvaluationDurationMs, scope));
+    }
+    return metrics;
+}
+export function toAgentSuccessRunMetrics(analysis) {
+    const metrics = [];
+    for (const aggregate of analysis.aggregates) {
+        const scope = { variantId: aggregate.treatmentId };
+        metrics.push(toExperimentMetric(availableMetric("evaluableCaseCount", aggregate.evaluableCaseCount, "count"), scope), toExperimentMetric(availableMetric("successfulCaseCount", aggregate.successfulCaseCount, "count"), scope), toExperimentMetric(aggregate.taskSuccessRate, scope), toExperimentMetric(aggregate.initialAttemptSuccessRate, scope));
+        for (const mean of Object.values(aggregate.means))
+            metrics.push(toExperimentMetric(mean.metric, scope));
+        const repair = aggregate.repair;
+        if (repair) {
+            metrics.push(toExperimentMetric(availableMetric("initialAttemptSuccessfulCount", repair.initialAttemptSuccessfulCount, "count"), scope), toExperimentMetric(repair.finalTaskSuccessRate, scope), toExperimentMetric(availableMetric("finalSuccessfulCount", repair.finalSuccessfulCount, "count"), scope), toExperimentMetric(availableMetric("repairAttemptedCaseCount", repair.repairAttemptedCaseCount, "count"), scope), toExperimentMetric(availableMetric("repairedCaseCount", repair.repairedCaseCount, "count"), scope), toExperimentMetric(repair.repairSuccessRate, scope), toExperimentMetric(repair.meanAttemptsPerEvaluableCase, scope));
+        }
+    }
+    return metrics;
+}

package/dist/src/experiments/plugins/agentSuccessRate/plugin.js

@@ -0,0 +1,507 @@
+import { access, mkdir, readFile, rm, rmdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { runMeasuredCommand } from "../../../core/runMeasuredCommand.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { buildMyDevKitIndex, probeMyDevKitVersion } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { analyzeAgentSuccessRate } from "./analysis.js";
+import { projectAgentFacingTask } from "./agentTaskProjection.js";
+import { AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE, buildAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateArtifactFamily } from "./analysisArtifact.js";
+import { agentSuccessRateConfigDefinition, defaultAgentSuccessRateConfig, validateAgentSuccessRateConfig } from "./config.js";
+import { AgentSuccessPackContextBuilder, DEFAULT_AGENT_SUCCESS_KIT_COMMAND, buildContextTaskInput, buildRawFullFileContext, collectForbiddenAgentValues } from "./contextGeneration.js";
+import { AGENT_SUCCESS_RATE_SELF_ONLY_MESSAGE, AgentSuccessRateInputError, boundMessage, executeAgentSuccessCase, readAgentSuccessTasksInput, resolveControlledBenchmarkProject, resolveSandboxRuntimeRoot, selectAgentSuccessTasks } from "./execution.js";
+import { AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE, buildAgentSuccessRateExecutionArtifact, validateAgentSuccessRateExecutionArtifact } from "./executionArtifact.js";
+import { attemptEvidenceOf } from "./attemptEvidence.js";
+import { executeAgentSuccessRepairCase } from "./repairExecution.js";
+import { AGENT_SUCCESS_RATE_EXECUTION_MODE, AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_VARIANTS, agentSuccessRateMetadata } from "./metadata.js";
+import { toAgentSuccessOutcomeMetrics, toAgentSuccessRunMetrics } from "./metrics.js";
+import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, createRealAgentProposalSource } from "./realAgentExecution.js";
+export const AGENT_SUCCESS_RATE_PERSISTENCE_FAILURE_MESSAGE = "Agent success rate artifact persistence failed.";
+export const AGENT_SUCCESS_RATE_ARTIFACT_STATE_MESSAGE = "Agent success rate artifact state was inconsistent; nothing was written.";
+const defaultArtifactIo = {
+    ensureDirectory: async (directory) => {
+        await mkdir(directory, { recursive: true });
+    },
+    exists: async (filePath) => {
+        try {
+            await access(filePath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    },
+    writeFile: (filePath, content) => writeFile(filePath, content, "utf8"),
+    removeFile: (filePath) => rm(filePath, { force: true })
+};
+export const agentSuccessRatePlugin = {
+    metadata: agentSuccessRateMetadata,
+    defaultConfig: defaultAgentSuccessRateConfig,
+    configDefinition: agentSuccessRateConfigDefinition,
+    supportedVariants: AGENT_SUCCESS_RATE_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateAgentSuccessRateConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        // The runner does not enforce supportedTargets; refuse before any filesystem mutation.
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            throw new AgentSuccessRateInputError(AGENT_SUCCESS_RATE_SELF_ONLY_MESSAGE);
+        }
+        const inputs = context.inputs;
+        // Fail closed on inputs and project resolution before any sandbox exists.
+        const real = readRealAgentConfig(context.config);
+        const loadedTasks = selectAgentSuccessTasks(readAgentSuccessTasksInput(inputs, { requireFixture: real === null }), context.config);
+        // Real-agent mode seals each task: the fixture is captured only as a leak-guard value and then removed, so nothing
+        // downstream can read the reference patch.
+        const forbiddenByTask = new Map();
+        const tasks = real === null ? loadedTasks : loadedTasks.map((task) => sealTask(task, forbiddenByTask));
+        const executionMode = real === null ? AGENT_SUCCESS_RATE_EXECUTION_MODE : AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE;
+        const projectRoots = new Map();
+        for (const task of tasks) {
+            if (!projectRoots.has(task.benchmarkProject))
+                projectRoots.set(task.benchmarkProject, await resolveControlledBenchmarkProject(context.toolRoot, task.benchmarkProject));
+        }
+        const outDir = path.resolve(context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir));
+        const runtimeRoot = await resolveSandboxRuntimeRoot(outDir, context.toolRoot);
+        const io = readArtifactIo(inputs);
+        const removeAttemptDirectory = readRemoveAttemptDirectoryInput(inputs);
+        const dependencies = readDependenciesInput(inputs);
+        await io.ensureDirectory(outDir);
+        const runtimeRootExisted = await io.exists(runtimeRoot);
+        const caseEvidence = [];
+        const patchFiles = [];
+        const contextFiles = [];
+        const warnings = [];
+        const packBuilder = real === null
+            ? null
+            : new AgentSuccessPackContextBuilder(real.kitCommand, { ...defaultContextDependencies(), ...readContextDependenciesInput(inputs) });
+        const settings = real === null
+            ? null
+            : {
+                providerId: real.providerId,
+                timeoutMs: real.timeoutMs,
+                env: readAgentEnvInput(inputs),
+                runAgent: readRunAgentInput(inputs),
+                outDir,
+                protectedRoots: [context.toolRoot, outDir, runtimeRoot, packageRoot(), ...projectRoots.values()],
+                privateRoots: [outDir, context.toolRoot],
+                ...readRemoveDirectoryInput(inputs)
+            };
+        try {
+            for (const task of tasks) {
+                const canonicalProjectRoot = projectRoots.get(task.benchmarkProject);
+                let repairContexts = null;
+                if (real !== null && settings !== null && packBuilder !== null) {
+                    // Both treatment contexts come from the same clean canonical project state, before any sandbox exists.
+                    const contextInput = buildContextTaskInput(task);
+                    const contexts = {
+                        "raw-full-file": await buildRawFullFileContext(contextInput, canonicalProjectRoot),
+                        "context-pack": await packBuilder.build(contextInput, canonicalProjectRoot)
+                    };
+                    repairContexts = contexts;
+                }
+                const result = real !== null && settings !== null && repairContexts !== null
+                    ? await executeAgentSuccessRepairCase({
+                        task,
+                        runId: context.runId,
+                        canonicalProjectRoot,
+                        runtimeRoot,
+                        privateRoots: [outDir, context.toolRoot],
+                        dependencies,
+                        // Each attempt is built from the same public task, the same original treatment context and (for repairs)
+                        // only the harness-authored feedback about the previous attempt.
+                        repairFor: () => ({
+                            providerId: real.providerId,
+                            repairAttempts: real.repairAttempts,
+                            createProposalSource: ({ attemptNumber, feedback }) => createRealAgentProposalSource({
+                                settings,
+                                runId: context.runId,
+                                task: projectAgentFacingTask(task),
+                                contexts: repairContexts,
+                                forbidden: forbiddenByTask.get(task.id) ?? [],
+                                attemptNumber,
+                                ...(feedback ? { repair: { feedback, maxAttempts: 1 + real.repairAttempts } } : {})
+                            })
+                        })
+                    })
+                    : await executeAgentSuccessCase({
+                        task,
+                        runId: context.runId,
+                        canonicalProjectRoot,
+                        runtimeRoot,
+                        privateRoots: [outDir, context.toolRoot],
+                        dependencies
+                    });
+                caseEvidence.push(result.evidence);
+                patchFiles.push(...result.patchFiles);
+                contextFiles.push(...result.contextFiles);
+            }
+        }
+        finally {
+            // Only an empty runtime root this run created is removed; rmdir refuses a non-empty directory.
+            if (!runtimeRootExisted) {
+                // rmdir refuses a non-empty directory, so a leftover sandbox keeps the root; that is reported, never hidden.
+                await rmdir(runtimeRoot).catch((error) => {
+                    warnings.push({ code: "sandbox-runtime-root-not-removed", message: boundMessage(`The sandbox runtime root could not be removed: ${error instanceof Error ? error.message : String(error)}`, [outDir, context.toolRoot]) });
+                });
+            }
+            const disposeReason = packBuilder === null ? null : await packBuilder.dispose();
+            if (disposeReason !== null)
+                warnings.push({ code: "context-work-directory-not-removed", message: disposeReason });
+        }
+        // Scientific analysis is calculated once, in memory, before persistence; reports never recalculate it.
+        const analysis = analyzeAgentSuccessRate(tasks, caseEvidence, executionMode);
+        const completedAt = new Date().toISOString();
+        const common = { runId: context.runId, pluginId: AGENT_SUCCESS_RATE_PLUGIN_ID, pluginSchemaVersion: agentSuccessRateMetadata.schemaVersion, startedAt, completedAt };
+        const executionArtifact = buildAgentSuccessRateExecutionArtifact({
+            ...common,
+            cases: caseEvidence,
+            realAgent: real === null
+                ? undefined
+                : {
+                    providerId: real.providerId,
+                    timeoutMs: real.timeoutMs,
+                    attemptsPerTreatment: (1 + real.repairAttempts),
+                    repairAttempts: real.repairAttempts,
+                    promptTransport: "stdin",
+                    contextEffectEvaluated: analysis.contextEffectEvaluated
+                }
+        });
+        const analysisArtifact = buildAgentSuccessRateAnalysisArtifact({ ...common, analysis });
+        const problems = [
+            ...validateAgentSuccessRateExecutionArtifact(executionArtifact),
+            ...validateAgentSuccessRateAnalysisArtifact(analysisArtifact),
+            ...validateAgentSuccessRateArtifactFamily(executionArtifact, analysisArtifact),
+            ...validatePatchFiles(caseEvidence, patchFiles),
+            ...validateContextFiles(caseEvidence, contextFiles),
+            ...(await validateAgentArtifacts(caseEvidence, outDir, io))
+        ];
+        if (problems.length > 0) {
+            throw new Error(AGENT_SUCCESS_RATE_ARTIFACT_STATE_MESSAGE + (await removeAgentAttemptDirectories(caseEvidence, outDir, removeAttemptDirectory)));
+        }
+        // Context and patch artifacts, then execution, then analysis last. Pre-existing files are never overwritten.
+        const targets = [
+            ...contextFiles.map((file) => ({ filePath: resolveWithinRoot(outDir, file.relativePath), content: file.content })),
+            ...patchFiles.map((file) => ({ filePath: resolveWithinRoot(outDir, file.relativePath), content: file.content })),
+            { filePath: resolveWithinRoot(outDir, AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE), content: `${JSON.stringify(executionArtifact, null, 2)}\n` },
+            { filePath: resolveWithinRoot(outDir, AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE), content: `${JSON.stringify(analysisArtifact, null, 2)}\n` }
+        ];
+        for (const target of targets) {
+            if (await io.exists(target.filePath)) {
+                throw new Error(AGENT_SUCCESS_RATE_PERSISTENCE_FAILURE_MESSAGE + (await removeAgentAttemptDirectories(caseEvidence, outDir, removeAttemptDirectory)));
+            }
+        }
+        const created = [];
+        try {
+            for (const target of targets) {
+                await io.ensureDirectory(path.dirname(target.filePath));
+                await io.writeFile(target.filePath, target.content);
+                created.push(target.filePath);
+            }
+        }
+        catch {
+            // Remove only files this attempt created; never touch anything else.
+            for (const filePath of created.reverse()) {
+                try {
+                    await io.removeFile(filePath);
+                }
+                catch {
+                    // best effort: the failure below is the outcome
+                }
+            }
+            throw new Error(AGENT_SUCCESS_RATE_PERSISTENCE_FAILURE_MESSAGE + (await removeAgentAttemptDirectories(caseEvidence, outDir, removeAttemptDirectory)));
+        }
+        return mapAgentSuccessRateToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            analysis,
+            patchArtifactPaths: patchFiles.map((file) => file.relativePath),
+            contextArtifactPaths: contextFiles.map((file) => file.relativePath),
+            providerId: real?.providerId ?? null,
+            realSettings: real === null ? undefined : { timeoutMs: real.timeoutMs, repairAttempts: real.repairAttempts },
+            warnings
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+/** Every patch artifact must be promised by exactly one treatment's evidence, and vice versa. */
+function validatePatchFiles(caseEvidence, patchFiles) {
+    const problems = [];
+    const promised = caseEvidence
+        .flatMap((entry) => entry.treatments.flatMap((t) => attemptEvidenceOf(t).flatMap((attempt) => [attempt.proposedPatchPath, attempt.appliedPatchPath])))
+        .filter((p) => p !== null);
+    const produced = patchFiles.map((file) => file.relativePath);
+    if (new Set(produced).size !== produced.length)
+        problems.push("duplicate patch artifact path.");
+    if ([...promised].sort().join("\n") !== [...produced].sort().join("\n"))
+        problems.push("patch artifacts do not match the paths promised by execution evidence.");
+    return problems;
+}
+export function mapAgentSuccessRateToRun(args) {
+    const realMode = args.analysis.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE;
+    const patchOwner = new Map();
+    for (const entry of args.caseEvidence) {
+        for (const t of entry.treatments) {
+            for (const attempt of attemptEvidenceOf(t)) {
+                for (const p of [attempt.proposedPatchPath, attempt.appliedPatchPath])
+                    if (p)
+                        patchOwner.set(p, { caseId: entry.caseId, variantId: t.treatmentId });
+            }
+        }
+    }
+    const cases = args.caseEvidence.map((entry, caseIndex) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: entry.treatments.map((treatment, treatmentIndex) => buildOutcome(entry, treatment, args.analysis.cases[caseIndex].treatments[treatmentIndex], args.analysis.executionMode, args.analysis.contextEffectEvaluated)),
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality, ...(realMode ? {} : { fixtureId: entry.fixtureId }) }
+    }));
+    const run = {
+        runId: args.runId,
+        pluginId: AGENT_SUCCESS_RATE_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateTreatmentStatus(args.caseEvidence.flatMap((entry) => entry.treatments.map((treatment) => treatment.status))),
+        target: args.target,
+        variants: AGENT_SUCCESS_RATE_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toAgentSuccessRunMetrics(args.analysis),
+        artifacts: [
+            {
+                id: "agent-success-rate-execution",
+                label: "Agent success rate execution evidence",
+                path: AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case, per-treatment execution evidence without source text, patch bodies or raw command output."
+            },
+            {
+                id: "agent-success-rate-analysis",
+                label: "Agent success rate scientific analysis",
+                path: AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Task-success, edit-quality and blast-radius metrics with explicit availability, plus matched-case aggregates."
+            },
+            ...(args.contextArtifactPaths ?? []).map((contextPath) => ({
+                id: `context:${contextPath}`,
+                label: "Treatment source context supplied to the provider",
+                path: contextPath,
+                kind: "artifact",
+                mimeType: "text/plain"
+            })),
+            ...args.patchArtifactPaths.map((patchPath) => ({
+                id: `patch:${patchPath}`,
+                label: patchLabel(patchPath, realMode),
+                path: patchPath,
+                kind: "artifact",
+                mimeType: "text/x-diff",
+                ...patchOwner.get(patchPath)
+            }))
+        ],
+        warnings: [...(args.warnings ?? [])],
+        // Outcome failures are the authoritative per-treatment failure location.
+        failures: [],
+        metadata: {
+            executionMode: args.analysis.executionMode,
+            ...(args.providerId ? { providerId: args.providerId } : {}),
+            ...(args.realSettings ? { timeoutMs: args.realSettings.timeoutMs, repairAttempts: args.realSettings.repairAttempts } : {}),
+            contextEffectEvaluated: args.analysis.contextEffectEvaluated,
+            executionArtifactPath: AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE,
+            analysisArtifactPath: AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE
+        },
+        caseExecutionEvidence: structuredClone(args.caseEvidence),
+        analysis: structuredClone(args.analysis)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment, treatmentAnalysis, executionMode, contextEffectEvaluated) {
+    const { caseId } = caseEvidence;
+    const { treatmentId } = treatment;
+    const success = treatmentAnalysis.metrics.taskSuccess;
+    return {
+        id: `${caseId}:${treatmentId}`,
+        caseId,
+        variantId: treatmentId,
+        status: treatment.status,
+        metrics: toAgentSuccessOutcomeMetrics(treatmentAnalysis),
+        artifacts: [],
+        warnings: treatment.availability === "baseline-invalid"
+            ? [
+                {
+                    code: "baseline-invalid",
+                    message: "The benchmark baseline is not evaluable; no implementation attempt was scored.",
+                    variantId: treatmentId,
+                    caseId,
+                    details: { reasons: treatment.baselineAssessment?.reasons ?? [] }
+                }
+            ]
+            : [],
+        failures: treatment.errors.map((error) => ({ code: error.code, message: error.message, variantId: treatmentId, caseId, recoverable: false })),
+        metadata: {
+            evidenceAvailability: treatment.availability,
+            // taskSuccess is a measurement, not an infrastructure status; null means it could not be determined.
+            taskSuccess: success.availability === "available" ? success.value : null,
+            executionMode,
+            ...(treatment.realAgent ? { providerId: treatment.realAgent.providerId, providerStatus: treatment.realAgent.providerStatus } : {}),
+            ...(treatmentAnalysis.repair
+                ? {
+                    attemptCount: treatmentAnalysis.repair.attemptCount,
+                    repairAttemptCount: treatmentAnalysis.repair.repairAttemptCount,
+                    initialAttemptTaskSuccess: treatmentAnalysis.repair.initialAttemptTaskSuccess.availability === "available" ? treatmentAnalysis.repair.initialAttemptTaskSuccess.value : null,
+                    repairSucceeded: treatmentAnalysis.repair.repairSucceeded.availability === "available" ? treatmentAnalysis.repair.repairSucceeded.value : null
+                }
+                : {}),
+            contextEffectEvaluated
+        }
+    };
+}
+function aggregateTreatmentStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    if (statuses.every((status) => status === "completed"))
+        return "completed";
+    if (statuses.every((status) => status === "skipped"))
+        return "skipped";
+    if (statuses.every((status) => status === "failed"))
+        return "failed";
+    return "partial";
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.agentSuccessDependencies;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}
+function readArtifactIo(inputs) {
+    const value = inputs?.agentSuccessArtifactIo;
+    return value && typeof value === "object" && !Array.isArray(value) ? { ...defaultArtifactIo, ...value } : defaultArtifactIo;
+}
+/** Real-agent mode exists only when a provider is named; the closed config validator has already enforced the opt-in. */
+function readRealAgentConfig(config) {
+    if (config.agentId === undefined)
+        return null;
+    return {
+        providerId: config.agentId,
+        timeoutMs: config.timeoutMs ?? AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS,
+        kitCommand: config.kitCommand ?? DEFAULT_AGENT_SUCCESS_KIT_COMMAND,
+        repairAttempts: config.repairAttempts ?? 0
+    };
+}
+/** Captures hidden values for the leak guard, then returns a copy of the task with the reference fixture removed. */
+function sealTask(task, forbiddenByTask) {
+    forbiddenByTask.set(task.id, collectForbiddenAgentValues(task));
+    const sealed = { ...task };
+    delete sealed.deterministicFixture;
+    return sealed;
+}
+/** The installed package root (two levels above src/ or dist/ output of this module). */
+function packageRoot() {
+    return fileURLToPath(new URL("../../../../", import.meta.url));
+}
+function defaultContextDependencies() {
+    return {
+        buildIndex: buildMyDevKitIndex,
+        runCommand: runMeasuredCommand,
+        readSymbolIndex: async (indexDir) => JSON.parse(await readFile(path.join(indexDir, "symbol-index.json"), "utf8")),
+        probeVersion: probeMyDevKitVersion
+    };
+}
+function readContextDependenciesInput(inputs) {
+    const value = inputs?.agentSuccessContextDependencies;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+function readRunAgentInput(inputs) {
+    const value = inputs?.agentSuccessRunAgent;
+    return typeof value === "function" ? value : runAgentPrompt;
+}
+function readAgentEnvInput(inputs) {
+    const value = inputs?.agentSuccessAgentEnv;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}
+/** Every context artifact must be promised by exactly one treatment's evidence, and vice versa. */
+function validateContextFiles(caseEvidence, contextFiles) {
+    // The treatment context is persisted once, by the initial attempt, and repairs reuse it unchanged.
+    const promised = caseEvidence.flatMap((entry) => entry.treatments.map((t) => attemptEvidenceOf(t)[0]?.realAgent?.context.contextArtifactPath ?? null)).filter((p) => p !== null);
+    const produced = contextFiles.map((file) => file.relativePath);
+    const problems = [];
+    if (new Set(produced).size !== produced.length)
+        problems.push("duplicate context artifact path.");
+    if ([...promised].sort().join("\n") !== [...produced].sort().join("\n"))
+        problems.push("context artifacts do not match the paths promised by execution evidence.");
+    return problems;
+}
+/** Agent attempt files are written by the shared agent runner; each referenced file must exist and attempts are single. */
+async function validateAgentArtifacts(caseEvidence, outDir, io) {
+    const problems = [];
+    const directories = new Set();
+    for (const entry of caseEvidence) {
+        for (const treatment of entry.treatments) {
+            for (const attempt of attemptEvidenceOf(treatment)) {
+                const real = attempt.realAgent;
+                if (!real?.agentArtifactDirectory)
+                    continue;
+                if (directories.has(real.agentArtifactDirectory))
+                    problems.push("duplicate agent attempt directory.");
+                directories.add(real.agentArtifactDirectory);
+                if (!real.agentArtifactDirectory.endsWith(`/attempt-${real.attempt}`))
+                    problems.push(`agent attempt directory is not attempt-${real.attempt}.`);
+                for (const reference of Object.values(real.agentArtifacts)) {
+                    if (reference === null)
+                        continue;
+                    try {
+                        if (!(await io.exists(resolveWithinRoot(outDir, reference))))
+                            problems.push(`agent artifact ${reference} is missing.`);
+                    }
+                    catch {
+                        problems.push("agent artifact path is invalid.");
+                    }
+                }
+            }
+        }
+    }
+    return problems;
+}
+/**
+ * Removes only attempt directories this run created (those recorded as invoked). Removal is best effort, but a directory
+ * that could not be removed is counted and reported in the returned suffix so cleanup is never presented as complete.
+ */
+async function removeAgentAttemptDirectories(caseEvidence, outDir, removeDirectory) {
+    let failures = 0;
+    for (const entry of caseEvidence) {
+        for (const treatment of entry.treatments) {
+            for (const attempt of attemptEvidenceOf(treatment)) {
+                const directory = attempt.realAgent?.agentArtifactDirectory;
+                if (!directory)
+                    continue;
+                try {
+                    await removeDirectory(resolveWithinRoot(outDir, directory));
+                }
+                catch {
+                    failures += 1;
+                }
+            }
+        }
+    }
+    return failures > 0 ? ` ${failures} agent attempt director${failures === 1 ? "y" : "ies"} could not be removed.` : "";
+}
+function patchLabel(patchPath, realMode) {
+    const attempt = /attempt-(\d+)-/.exec(patchPath)?.[1];
+    const suffix = realMode && attempt && attempt !== "1" ? ` (attempt ${attempt})` : "";
+    if (patchPath.endsWith("-applied.patch"))
+        return `Applied patch (from change evidence)${suffix}`;
+    return `${realMode ? "Proposed patch (provider final answer)" : "Proposed deterministic fixture patch"}${suffix}`;
+}
+/** Removal seam for failed-persistence attempt directories so a removal failure can be proven on every platform. */
+function readRemoveAttemptDirectoryInput(inputs) {
+    const value = inputs?.agentSuccessRemoveAttemptDirectory;
+    return typeof value === "function" ? value : (directory) => rm(directory, { recursive: true, force: true });
+}
+function readRemoveDirectoryInput(inputs) {
+    const value = inputs?.agentSuccessRemoveDirectory;
+    return typeof value === "function" ? { removeDirectory: value } : {};
+}