@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/experiments/plugins/agentSuccessRate/execution.js

@@ -0,0 +1,441 @@
+import { createHash } from "node:crypto";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { applyPatchToSandbox, assessBaseline, runVerificationChecks, validateAgentSuccessTask } from "../../../evaluation/agentSuccess/index.js";
+import { BenchmarkSandboxError, createBenchmarkSandbox, removeBenchmarkSandbox, snapshotProjectTree } from "../../../evaluation/benchmarkSandbox/index.js";
+import { isSameOrInside, resolvePhysicalPath } from "../../../evaluation/benchmarkSandbox/pathPolicy.js";
+import { captureChangeSet } from "../../../evaluation/changeSet/index.js";
+import { isPathInside, sanitizePathSegment } from "../../outputPaths.js";
+import { AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+export const AGENT_SUCCESS_RATE_MISSING_INPUT_MESSAGE = "Agent success rate requires an agentSuccessTasks input (an array of AgentSuccessTaskV1). No production task catalog is bundled in this version.";
+export const AGENT_SUCCESS_RATE_SELF_ONLY_MESSAGE = "agent-success-rate supports only the self target; external-local targets are not supported.";
+const MAX_BOUNDED_MESSAGE = 300;
+const SAFE_PROJECT_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
+export class AgentSuccessRateInputError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "AgentSuccessRateInputError";
+    }
+}
+export const defaultAgentSuccessRateDependencies = {
+    createSandbox: createBenchmarkSandbox,
+    removeSandbox: removeBenchmarkSandbox,
+    applyPatch: applyPatchToSandbox,
+    captureChanges: captureChangeSet,
+    runChecks: (options) => runVerificationChecks(options)
+};
+// ---------------------------------------------------------------------------------------------------------------
+// Inputs
+// ---------------------------------------------------------------------------------------------------------------
+/**
+ * Validates every candidate with the Batch 1 validator before anything touches the filesystem. Fails closed: there is
+ * no default catalog, and no task is inferred from retrieval EvaluationCase objects.
+ */
+export function readAgentSuccessTasksInput(inputs, options = {}) {
+    const requireFixture = options.requireFixture !== false;
+    const value = inputs?.agentSuccessTasks;
+    if (value === undefined)
+        throw new AgentSuccessRateInputError(AGENT_SUCCESS_RATE_MISSING_INPUT_MESSAGE);
+    if (!Array.isArray(value))
+        throw new AgentSuccessRateInputError("agentSuccessTasks input must be an array of AgentSuccessTaskV1.");
+    if (value.length === 0)
+        throw new AgentSuccessRateInputError("agentSuccessTasks input must not be empty.");
+    const tasks = [];
+    const seen = new Set();
+    value.forEach((candidate, index) => {
+        const result = validateAgentSuccessTask(candidate);
+        if (!result.ok) {
+            const detail = result.issues
+                .slice(0, 5)
+                .map((issue) => `${issue.path || "(root)"}: ${issue.code}`)
+                .join("; ");
+            throw new AgentSuccessRateInputError(`agentSuccessTasks[${index}] is not a valid AgentSuccessTaskV1: ${detail}`);
+        }
+        const task = result.task;
+        if (seen.has(task.id))
+            throw new AgentSuccessRateInputError(`Duplicate agent-success task id: ${task.id}`);
+        seen.add(task.id);
+        if (requireFixture && !task.deterministicFixture) {
+            throw new AgentSuccessRateInputError(`Task ${task.id} has no deterministicFixture; deterministic-fixture mode requires one.`);
+        }
+        tasks.push(task);
+    });
+    return tasks;
+}
+/** Keeps input order, applies case then project filters, and fails clearly for unknown IDs or an empty selection. */
+export function selectAgentSuccessTasks(tasks, filters) {
+    const knownCases = new Set(tasks.map((task) => task.id));
+    const missingCases = (filters.caseIds ?? []).filter((caseId) => !knownCases.has(caseId));
+    if (missingCases.length > 0)
+        throw new AgentSuccessRateInputError(`Agent-success task not found: ${missingCases.join(", ")}`);
+    const knownProjects = new Set(tasks.map((task) => task.benchmarkProject));
+    const missingProjects = (filters.benchmarkProjects ?? []).filter((project) => !knownProjects.has(project));
+    if (missingProjects.length > 0)
+        throw new AgentSuccessRateInputError(`Benchmark project not found: ${missingProjects.join(", ")}`);
+    const selected = tasks
+        .filter((task) => !filters.caseIds?.length || filters.caseIds.includes(task.id))
+        .filter((task) => !filters.benchmarkProjects?.length || filters.benchmarkProjects.includes(task.benchmarkProject));
+    if (selected.length === 0)
+        throw new AgentSuccessRateInputError("No agent-success tasks matched the requested filters.");
+    return selected;
+}
+/**
+ * Resolves `<toolRoot>/benchmarks/projects/<benchmarkProject>` physically. A project ID that is not a plain path
+ * segment, a missing project, a link, or a project that resolves outside the controlled directory is rejected.
+ */
+export async function resolveControlledBenchmarkProject(toolRoot, benchmarkProject) {
+    if (!SAFE_PROJECT_ID.test(benchmarkProject)) {
+        throw new AgentSuccessRateInputError(`Benchmark project id is not a safe path segment: ${boundMessage(benchmarkProject, [])}`);
+    }
+    const projectsRoot = path.join(path.resolve(toolRoot), "benchmarks", "projects");
+    const candidate = resolveWithinRoot(projectsRoot, benchmarkProject);
+    let stats;
+    try {
+        stats = await lstat(candidate);
+    }
+    catch {
+        throw new AgentSuccessRateInputError(`Benchmark project not found in the controlled benchmark directory: ${benchmarkProject}`);
+    }
+    if (stats.isSymbolicLink() || !stats.isDirectory()) {
+        throw new AgentSuccessRateInputError(`Benchmark project must be a real directory: ${benchmarkProject}`);
+    }
+    const physicalRoot = await realpath(projectsRoot);
+    const physicalProject = await realpath(candidate);
+    if (physicalProject === physicalRoot || !isPathInside(physicalRoot, physicalProject)) {
+        throw new AgentSuccessRateInputError(`Benchmark project escapes the controlled benchmark directory: ${benchmarkProject}`);
+    }
+    return physicalProject;
+}
+/** The runtime root lives under the experiment output root and must never overlap the controlled benchmarks. */
+export async function resolveSandboxRuntimeRoot(outDir, toolRoot) {
+    const runtimeRoot = path.join(path.resolve(outDir), "sandboxes");
+    const physicalRuntime = await resolvePhysicalPath(runtimeRoot);
+    const physicalBenchmarks = await resolvePhysicalPath(path.join(path.resolve(toolRoot), "benchmarks"));
+    if (isSameOrInside(physicalBenchmarks, physicalRuntime) || isSameOrInside(physicalRuntime, physicalBenchmarks)) {
+        throw new AgentSuccessRateInputError("The experiment output root must not overlap the controlled benchmark directory.");
+    }
+    return runtimeRoot;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Execution
+// ---------------------------------------------------------------------------------------------------------------
+/** Attempt 1 keeps its original identity; every repair attempt derives a distinct sandbox id so no sandbox is ever reused. */
+export function deriveAgentSuccessSandboxId(runId, caseId, treatmentId, attemptNumber = 1) {
+    const attemptKey = attemptNumber === 1 ? "" : `\0attempt-${attemptNumber}`;
+    const digest = createHash("sha256").update(`${runId}\0${caseId}\0${treatmentId}${attemptKey}`).digest("hex").slice(0, 20);
+    return `asr-${digest}-${treatmentId === "raw-full-file" ? "raw" : "ctx"}${attemptNumber === 1 ? "" : `-a${attemptNumber}`}`;
+}
+export function agentSuccessPatchArtifactPath(benchmarkProject, caseId, treatmentId, kind, attemptNumber = 1) {
+    return `diffs/${sanitizePathSegment(benchmarkProject)}/${sanitizePathSegment(caseId)}/${sanitizePathSegment(treatmentId)}/attempt-${attemptNumber}-${kind}.patch`;
+}
+/** Bounds a message and replaces any known machine-local root so no absolute path is persisted. */
+export function boundMessage(message, privateRoots) {
+    let text = message;
+    for (const root of [...privateRoots].filter(Boolean).sort((a, b) => b.length - a.length)) {
+        for (const variant of new Set([root, root.replace(/\\/g, "/")]))
+            text = text.split(variant).join("<path>");
+    }
+    text = text.replace(/\s+/g, " ").trim();
+    return text.length > MAX_BOUNDED_MESSAGE ? `${text.slice(0, MAX_BOUNDED_MESSAGE)}...` : text;
+}
+function summarizeVerification(phase, privateRoots) {
+    const summarize = (result, checkClass) => ({
+        checkId: result.checkId,
+        class: checkClass,
+        status: result.status,
+        exitCode: result.exitCode,
+        durationMs: result.durationMs,
+        failureReason: result.failureReason === null ? null : boundMessage(result.failureReason, privateRoots)
+    });
+    return {
+        phase: phase.phase,
+        taskResults: phase.taskResults.map((result) => summarize(result, "task")),
+        regressionResults: phase.regressionResults.map((result) => summarize(result, "regression"))
+    };
+}
+function summarizeChange(changes) {
+    return {
+        baselineCommit: changes.baselineCommit,
+        changedFiles: changes.changedFiles.map((file) => ({ ...file })),
+        addedCount: changes.addedCount,
+        modifiedCount: changes.modifiedCount,
+        deletedCount: changes.deletedCount,
+        changedCount: changes.changedCount,
+        totalAdditions: changes.totalAdditions,
+        totalDeletions: changes.totalDeletions
+    };
+}
+/** Total line count of baseline files that are text (contain no NUL byte). Null when it cannot be determined reliably. */
+async function countBaselineTextLines(sandbox) {
+    try {
+        let lines = 0;
+        for (const entry of sandbox.baseline.manifest) {
+            const data = await readFile(path.join(sandbox.projectRoot, ...entry.path.split("/")));
+            if (data.includes(0))
+                continue;
+            for (const byte of data)
+                if (byte === 10)
+                    lines += 1;
+            if (data.length > 0 && data[data.length - 1] !== 10)
+                lines += 1;
+        }
+        return lines;
+    }
+    catch {
+        return null;
+    }
+}
+/** Compares every protected path with the sandbox baseline manifest after all Lab-run work is finished. */
+async function evaluateProtectedIntegrity(sandbox, task, changes) {
+    const protectedPaths = new Set(task.protectedFiles);
+    const mutated = new Set();
+    for (const file of changes?.changedFiles ?? [])
+        if (protectedPaths.has(file.relativePath))
+            mutated.add(file.relativePath);
+    try {
+        const working = new Map((await snapshotProjectTree(sandbox.projectRoot, { excludedNames: [".git"] })).map((entry) => [entry.path, entry.sha256]));
+        const baseline = new Map(sandbox.baseline.manifest.map((entry) => [entry.path, entry.sha256]));
+        for (const protectedPath of protectedPaths) {
+            if (working.get(protectedPath) !== baseline.get(protectedPath))
+                mutated.add(protectedPath);
+        }
+    }
+    catch {
+        return { status: mutated.size > 0 ? "mutated" : "unproven", mutatedPaths: [...mutated].sort() };
+    }
+    return { status: mutated.size > 0 ? "mutated" : "intact", mutatedPaths: [...mutated].sort() };
+}
+function emptyPatchEvidence() {
+    return { attempted: false, outcome: null, code: null, message: null, appliedFiles: [], rejections: [], attemptedProtectedPaths: [], proposedPatchBytes: 0 };
+}
+function recordPatchOutcome(evidence, result, privateRoots) {
+    evidence.outcome = result.outcome;
+    switch (result.outcome) {
+        case "parse-failure":
+            evidence.code = result.code;
+            break;
+        case "policy-rejection":
+            evidence.rejections = result.rejections.map((rejection) => ({ code: rejection.code, path: rejection.path ?? null }));
+            evidence.attemptedProtectedPaths = [
+                ...new Set(result.rejections.filter((rejection) => rejection.code === "PROTECTED_PATH" && rejection.path).map((rejection) => rejection.path))
+            ].sort();
+            break;
+        case "git-check-failure":
+        case "git-apply-failure":
+            evidence.message = boundMessage(result.message, privateRoots);
+            break;
+        case "success":
+            evidence.appliedFiles = result.files.map((file) => ({ path: file.path, status: file.status }));
+            break;
+    }
+}
+function isIndeterminate(result) {
+    return result.status === "timeout" || result.status === "error";
+}
+/**
+ * Runs one case/treatment: independent sandbox, baseline verification, the task's deterministic fixture patch through
+ * the Lab-owned pipeline, change capture, post-edit verification, protected-file integrity, and guarded cleanup.
+ * Cleanup is attempted after every successfully created sandbox.
+ */
+export async function executeAgentSuccessTreatment(args) {
+    const deps = { ...defaultAgentSuccessRateDependencies, ...args.dependencies };
+    const { task, treatmentId } = args;
+    const privateRoots = [args.runtimeRoot, args.canonicalProjectRoot, ...args.privateRoots];
+    const proposalSource = args.proposalSource;
+    const fixture = proposalSource ? undefined : task.deterministicFixture;
+    const attemptNumber = args.attemptNumber ?? 1;
+    const sandboxId = deriveAgentSuccessSandboxId(args.runId, task.id, treatmentId, attemptNumber);
+    const started = Date.now();
+    const evidence = {
+        treatmentId,
+        status: "failed",
+        availability: "infrastructure-failure",
+        sandboxId,
+        sandboxBaseline: null,
+        baselineTextLineCount: null,
+        baselineVerification: null,
+        baselineAssessment: null,
+        patch: emptyPatchEvidence(),
+        change: null,
+        postEditVerification: null,
+        protectedIntegrity: { status: "unproven", mutatedPaths: [] },
+        timing: { agentDurationMs: null, baselineVerificationDurationMs: null, patchPipelineDurationMs: null, postEditVerificationDurationMs: null, evaluationDurationMs: null },
+        agentTokenUsage: null,
+        cleanup: { attempted: false, removed: false, reason: null },
+        proposedPatchPath: null,
+        appliedPatchPath: null,
+        errors: []
+    };
+    const patchFiles = [];
+    const contextFiles = [];
+    const addError = (code, message) => {
+        evidence.errors.push({ code, message: boundMessage(message, privateRoots) });
+    };
+    if (!proposalSource && !fixture) {
+        addError("FIXTURE_MISSING", `Task ${task.id} has no deterministicFixture.`);
+        return { evidence, patchFiles, contextFiles };
+    }
+    /** Records that no provider ran, so real-agent evidence is never absent for a treatment. */
+    const recordNotInvoked = async (reason) => {
+        if (!proposalSource || evidence.realAgent)
+            return;
+        try {
+            const skipped = await proposalSource({ treatmentId, invoke: false, notInvokedReason: reason });
+            evidence.realAgent = skipped.realAgent;
+        }
+        catch {
+            // The treatment already carries its own failure evidence.
+        }
+    };
+    let sandbox;
+    try {
+        sandbox = await deps.createSandbox({ canonicalProjectRoot: args.canonicalProjectRoot, runtimeRoot: args.runtimeRoot, sandboxId });
+    }
+    catch (error) {
+        addError(error instanceof BenchmarkSandboxError ? `SANDBOX_${error.code}` : "SANDBOX_CREATION_FAILED", error instanceof Error ? error.message : String(error));
+        evidence.timing.evaluationDurationMs = Date.now() - started;
+        await recordNotInvoked("sandbox-unavailable");
+        return { evidence, patchFiles, contextFiles };
+    }
+    let changes = null;
+    try {
+        evidence.sandboxBaseline = { commit: sandbox.baseline.commit, fileCount: sandbox.baseline.fileCount, digest: sandbox.baseline.digest };
+        evidence.baselineTextLineCount = await countBaselineTextLines(sandbox);
+        const baselineStarted = Date.now();
+        const baseline = await deps.runChecks({ sandbox, task, phase: "baseline" });
+        evidence.timing.baselineVerificationDurationMs = Date.now() - baselineStarted;
+        evidence.baselineVerification = summarizeVerification(baseline, privateRoots);
+        const assessment = assessBaseline(baseline);
+        evidence.baselineAssessment = { evaluable: assessment.evaluable, reasons: [...assessment.reasons] };
+        if (!assessment.evaluable) {
+            evidence.status = "skipped";
+            evidence.availability = "baseline-invalid";
+            evidence.protectedIntegrity = await evaluateProtectedIntegrity(sandbox, task, null);
+            await recordNotInvoked("baseline-not-evaluable");
+        }
+        else {
+            // Exactly one proposal per treatment: the deterministic fixture, or one provider attempt (never both).
+            let proposalText = null;
+            let cwdCleanupFailed = false;
+            if (proposalSource) {
+                const proposal = await proposalSource({ treatmentId, invoke: true });
+                evidence.realAgent = proposal.realAgent;
+                evidence.timing.agentDurationMs = proposal.agentDurationMs;
+                evidence.agentTokenUsage = proposal.agentTokenUsage;
+                evidence.errors.push(...proposal.errors);
+                if (proposal.contextFile)
+                    contextFiles.push(proposal.contextFile);
+                if (!proposal.realAgent.cwdCleanup.removed && proposal.realAgent.cwdCleanup.attempted) {
+                    cwdCleanupFailed = true;
+                    addError("AGENT_CWD_CLEANUP_FAILED", proposal.realAgent.cwdCleanup.reason ?? "provider working directory cleanup failed.");
+                }
+                proposalText = proposal.kind === "proposal" ? proposal.text : null;
+            }
+            else {
+                proposalText = fixture.patch;
+            }
+            if (proposalText === null) {
+                // No usable provider answer: nothing is applied, scored or fabricated.
+                evidence.protectedIntegrity = await evaluateProtectedIntegrity(sandbox, task, null);
+                if (evidence.protectedIntegrity.status === "unproven")
+                    addError("PROTECTED_INTEGRITY_UNPROVEN", "Protected-file integrity could not be proven.");
+                evidence.status = "partial";
+                evidence.availability = "incomplete";
+                return { evidence, patchFiles, contextFiles };
+            }
+            const proposedPath = agentSuccessPatchArtifactPath(task.benchmarkProject, task.id, treatmentId, "proposed", attemptNumber);
+            patchFiles.push({ relativePath: proposedPath, content: proposalText });
+            evidence.proposedPatchPath = proposedPath;
+            evidence.patch.attempted = true;
+            evidence.patch.proposedPatchBytes = Buffer.byteLength(proposalText, "utf8");
+            const patchStarted = Date.now();
+            const applied = await deps.applyPatch({ sandbox, rawProposal: proposalText, protectedFiles: task.protectedFiles });
+            recordPatchOutcome(evidence.patch, applied, privateRoots);
+            let captureFailed = false;
+            if (applied.outcome === "success") {
+                try {
+                    changes = await deps.captureChanges({ sandbox });
+                    evidence.change = summarizeChange(changes);
+                    const appliedPath = agentSuccessPatchArtifactPath(task.benchmarkProject, task.id, treatmentId, "applied", attemptNumber);
+                    patchFiles.push({ relativePath: appliedPath, content: changes.diff });
+                    evidence.appliedPatchPath = appliedPath;
+                }
+                catch (error) {
+                    captureFailed = true;
+                    addError("CHANGE_CAPTURE_FAILED", error instanceof Error ? error.message : String(error));
+                }
+            }
+            evidence.timing.patchPipelineDurationMs = Date.now() - patchStarted;
+            let postIndeterminate = false;
+            if (applied.outcome === "success") {
+                const postStarted = Date.now();
+                const post = await deps.runChecks({ sandbox, task, phase: "post-edit" });
+                evidence.timing.postEditVerificationDurationMs = Date.now() - postStarted;
+                evidence.postEditVerification = summarizeVerification(post, privateRoots);
+                postIndeterminate = [...post.taskResults, ...post.regressionResults].some(isIndeterminate);
+                if (postIndeterminate)
+                    addError("POST_EDIT_CHECK_INDETERMINATE", "At least one post-edit check timed out or errored.");
+            }
+            evidence.protectedIntegrity = await evaluateProtectedIntegrity(sandbox, task, changes);
+            if (evidence.protectedIntegrity.status === "unproven")
+                addError("PROTECTED_INTEGRITY_UNPROVEN", "Protected-file integrity could not be proven.");
+            const complete = !captureFailed && !postIndeterminate && !cwdCleanupFailed && evidence.protectedIntegrity.status !== "unproven";
+            evidence.status = complete ? "completed" : "partial";
+            evidence.availability = complete ? "complete" : "incomplete";
+        }
+    }
+    catch (error) {
+        evidence.status = "failed";
+        evidence.availability = "infrastructure-failure";
+        addError("EXECUTION_FAILED", error instanceof Error ? error.message : String(error));
+        await recordNotInvoked("execution-failed");
+    }
+    finally {
+        evidence.timing.evaluationDurationMs = Date.now() - started;
+        evidence.cleanup.attempted = true;
+        try {
+            const removal = await deps.removeSandbox({ runtimeRoot: args.runtimeRoot, sandboxId });
+            evidence.cleanup.removed = removal.removed;
+            if (!removal.removed)
+                evidence.cleanup.reason = boundMessage(removal.reason, privateRoots);
+        }
+        catch (error) {
+            evidence.cleanup.removed = false;
+            evidence.cleanup.reason = boundMessage(error instanceof Error ? error.message : String(error), privateRoots);
+        }
+        if (!evidence.cleanup.removed) {
+            addError("CLEANUP_FAILED", evidence.cleanup.reason ?? "sandbox cleanup failed.");
+            if (evidence.status === "completed") {
+                evidence.status = "partial";
+                evidence.availability = "incomplete";
+            }
+        }
+    }
+    return { evidence, patchFiles, contextFiles };
+}
+/** Executes both mandatory treatments, in fixed order, each in its own sandbox. */
+export async function executeAgentSuccessCase(args) {
+    const treatments = [];
+    const patchFiles = [];
+    const contextFiles = [];
+    for (const treatmentId of AGENT_SUCCESS_RATE_TREATMENT_IDS) {
+        const result = await executeAgentSuccessTreatment({ ...args, treatmentId });
+        treatments.push(result.evidence);
+        patchFiles.push(...result.patchFiles);
+        contextFiles.push(...result.contextFiles);
+    }
+    return {
+        evidence: {
+            caseId: args.task.id,
+            caseName: args.task.title,
+            benchmarkProject: args.task.benchmarkProject,
+            taskLocality: args.task.taskLocality,
+            // Real-agent mode never reads the fixture, so it has no fixture identity.
+            fixtureId: args.proposalSource ? "" : (args.task.deterministicFixture?.id ?? ""),
+            treatments
+        },
+        patchFiles,
+        contextFiles
+    };
+}

package/dist/src/experiments/plugins/agentSuccessRate/executionArtifact.js

@@ -0,0 +1,126 @@
+import { AGENT_SUCCESS_MAX_TOTAL_ATTEMPTS } from "./executionTypes.js";
+import { AGENT_SUCCESS_RATE_EXECUTION_MODE, AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+export const AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE = "agent-success-rate-execution.json";
+export const AGENT_SUCCESS_RATE_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-rate-execution-v1";
+export function buildAgentSuccessRateExecutionArtifact(args) {
+    return {
+        schemaVersion: AGENT_SUCCESS_RATE_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        executionMode: args.realAgent ? AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE : AGENT_SUCCESS_RATE_EXECUTION_MODE,
+        contextEffectEvaluated: args.realAgent ? args.realAgent.contextEffectEvaluated : false,
+        treatmentOrder: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+        cases: structuredClone(args.cases),
+        ...(args.realAgent
+            ? {
+                realAgent: {
+                    providerId: args.realAgent.providerId,
+                    timeoutMs: args.realAgent.timeoutMs,
+                    attemptsPerTreatment: args.realAgent.attemptsPerTreatment,
+                    repairAttempts: args.realAgent.repairAttempts,
+                    promptTransport: "stdin"
+                }
+            }
+            : {})
+    };
+}
+/** Structural validation of an in-memory execution artifact. Returns problems; empty means valid. */
+export function validateAgentSuccessRateExecutionArtifact(artifact) {
+    const problems = [];
+    if (artifact.schemaVersion !== AGENT_SUCCESS_RATE_EXECUTION_SCHEMA_VERSION)
+        problems.push("execution artifact schemaVersion is not recognized.");
+    if (artifact.pluginId !== AGENT_SUCCESS_RATE_PLUGIN_ID)
+        problems.push("execution artifact pluginId does not match.");
+    const real = artifact.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE;
+    if (artifact.executionMode !== AGENT_SUCCESS_RATE_EXECUTION_MODE && !real)
+        problems.push("execution artifact executionMode is not recognized.");
+    if (real) {
+        if (!artifact.realAgent)
+            problems.push("real-agent execution artifact is missing the realAgent run settings.");
+        if (typeof artifact.contextEffectEvaluated !== "boolean")
+            problems.push("execution artifact contextEffectEvaluated must be a boolean.");
+    }
+    else {
+        if (artifact.contextEffectEvaluated !== false)
+            problems.push("execution artifact must state contextEffectEvaluated=false.");
+        if (artifact.realAgent !== undefined)
+            problems.push("deterministic-fixture execution artifact must not carry realAgent settings.");
+    }
+    if (artifact.treatmentOrder.join(",") !== AGENT_SUCCESS_RATE_TREATMENT_IDS.join(","))
+        problems.push("execution artifact treatmentOrder is not the fixed order.");
+    const seen = new Set();
+    for (const entry of artifact.cases) {
+        if (seen.has(entry.caseId))
+            problems.push(`duplicate execution case ${entry.caseId}.`);
+        seen.add(entry.caseId);
+        if (entry.treatments.map((t) => t.treatmentId).join(",") !== AGENT_SUCCESS_RATE_TREATMENT_IDS.join(",")) {
+            problems.push(`case ${entry.caseId} does not hold both treatments in fixed order.`);
+        }
+        const sandboxIds = entry.treatments.map((t) => t.sandboxId);
+        if (new Set(sandboxIds).size !== sandboxIds.length)
+            problems.push(`case ${entry.caseId} treatments share a sandbox id.`);
+        for (const treatment of entry.treatments) {
+            if (!real) {
+                if (treatment.timing.agentDurationMs !== null || treatment.agentTokenUsage !== null || treatment.realAgent !== undefined) {
+                    problems.push(`case ${entry.caseId} claims agent evidence in deterministic mode.`);
+                }
+                continue;
+            }
+            const evidence = treatment.realAgent;
+            if (!evidence) {
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} is missing real-agent evidence.`);
+                continue;
+            }
+            problems.push(...validateAttempts(entry.caseId, treatment, artifact.realAgent));
+            if (artifact.realAgent && evidence.providerId !== artifact.realAgent.providerId)
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} provider does not match the run provider.`);
+            if (evidence.context.contextMode !== treatment.treatmentId)
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} context mode does not match the treatment.`);
+            if (evidence.providerInvoked !== (evidence.agentArtifactDirectory !== null))
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} agent artifact directory disagrees with provider invocation.`);
+            if (!evidence.providerInvoked && (treatment.timing.agentDurationMs !== null && evidence.providerStatus === "not-invoked"))
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} has an agent duration without a provider attempt.`);
+            if (evidence.finalAnswerAvailable !== treatment.patch.attempted)
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} patch attempt disagrees with the provider final answer.`);
+            if (evidence.context.availability !== "unavailable" && evidence.providerInvoked && evidence.context.contextArtifactPath === null)
+                problems.push(`case ${entry.caseId}/${treatment.treatmentId} is missing its context artifact reference.`);
+        }
+    }
+    return problems;
+}
+/** Attempts are ordered, contiguous and one-based, bounded by the configured allowance, and never share a sandbox. */
+function validateAttempts(caseId, treatment, run) {
+    const problems = [];
+    const label = `case ${caseId}/${treatment.treatmentId}`;
+    const attempts = treatment.attempts;
+    // Evidence recorded without attempt history is a single legacy attempt and is validated by the checks above.
+    if (!attempts || attempts.length === 0)
+        return problems;
+    const allowed = Math.min(1 + (run?.repairAttempts ?? 0), AGENT_SUCCESS_MAX_TOTAL_ATTEMPTS);
+    if (attempts.length > allowed)
+        problems.push(`${label} executed ${attempts.length} attempts but at most ${allowed} are allowed.`);
+    attempts.forEach((attempt, index) => {
+        if (attempt.attemptNumber !== index + 1)
+            problems.push(`${label} attempt numbering is not contiguous from 1.`);
+        if (attempt.evidence.attempts !== undefined)
+            problems.push(`${label} attempt ${attempt.attemptNumber} nests further attempts.`);
+        if (attempt.evidence.realAgent && attempt.evidence.realAgent.attempt !== attempt.attemptNumber)
+            problems.push(`${label} attempt ${attempt.attemptNumber} disagrees with its provider evidence.`);
+        if (run && attempt.providerId !== run.providerId)
+            problems.push(`${label} attempt ${attempt.attemptNumber} provider does not match the run provider.`);
+        if (index > 0) {
+            const previous = attempts[index - 1];
+            if (!previous.repairEligible || previous.taskSuccess !== false)
+                problems.push(`${label} attempt ${attempt.attemptNumber} followed an attempt that was not repair-eligible.`);
+        }
+    });
+    const sandboxIds = attempts.map((attempt) => attempt.evidence.sandboxId);
+    if (new Set(sandboxIds).size !== sandboxIds.length)
+        problems.push(`${label} attempts share a sandbox id.`);
+    if (attempts[attempts.length - 1].evidence.sandboxId !== treatment.sandboxId)
+        problems.push(`${label} top-level evidence is not the final attempt.`);
+    return problems;
+}

package/dist/src/experiments/plugins/agentSuccessRate/executionTypes.js

@@ -0,0 +1,2 @@
+export const AGENT_SUCCESS_MAX_REPAIR_ATTEMPTS = 2;
+export const AGENT_SUCCESS_MAX_TOTAL_ATTEMPTS = 3;

package/dist/src/experiments/plugins/agentSuccessRate/index.js

@@ -0,0 +1,20 @@
+export * from "./analysis.js";
+export * from "./analysisArtifact.js";
+export * from "./analysisTypes.js";
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./executionTypes.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./types.js";
+export * from "./agentTaskProjection.js";
+export * from "./contextGeneration.js";
+export * from "./realAgentExecution.js";
+export * from "./realAgentPrompt.js";
+export * from "./attemptEvidence.js";
+export * from "./repairPolicy.js";
+export * from "./repairFeedback.js";
+export * from "./repairAnalysis.js";
+export * from "./repairExecution.js";

package/dist/src/experiments/plugins/agentSuccessRate/metadata.js

@@ -0,0 +1,28 @@
+export const AGENT_SUCCESS_RATE_PLUGIN_ID = "agent-success-rate";
+export const AGENT_SUCCESS_RATE_TREATMENT_IDS = ["raw-full-file", "context-pack"];
+/** Deterministic-fixture mode executes fixture patches only; no coding agent runs. It is the default mode. */
+export const AGENT_SUCCESS_RATE_EXECUTION_MODE = "deterministic-fixture";
+/** Real-agent mode runs an initial provider attempt (plus bounded repairs, when configured) per case/treatment and evaluates each patch the provider generates. */
+export const AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE = "real-agent";
+export const agentSuccessRateMetadata = {
+    id: AGENT_SUCCESS_RATE_PLUGIN_ID,
+    name: "Agent Success Rate",
+    description: "Evaluate implementation tasks on disposable benchmark copies with trusted verification checks. Deterministic-fixture mode validates the evaluation pipeline and does not measure agent or context performance.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self"],
+    supportedOutputs: ["json", "html", "text", "artifact"]
+};
+/** Exactly two mandatory treatments, in fixed order. There is no treatment selector. */
+export const AGENT_SUCCESS_RATE_VARIANTS = [
+    {
+        id: "raw-full-file",
+        name: "Raw full-file context",
+        description: "Treatment identity for raw full-file context. In deterministic-fixture mode no context is supplied to any agent."
+    },
+    {
+        id: "context-pack",
+        name: "Context pack",
+        description: "Treatment identity for a bounded context pack. In deterministic-fixture mode no context is supplied to any agent."
+    }
+];