@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1

package/dist/src/experiments/plugins/agentSuccessRate/campaignQueue.js

@@ -0,0 +1,11 @@
+import { validateAgentSuccessTask } from "../../../evaluation/agentSuccess/index.js";
+import { campaignAssert, canonicalCampaignJson } from "./campaignIdentity.js";
+import { AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+/** Selection is already established by selectAgentSuccessTasks. Never sort, interleave or drop a treatment. */
+export function buildCampaignQueue(tasks, treatments, limit) {
+    campaignAssert(tasks.length > 0 && tasks.length <= 1000 && new Set(tasks.map((t) => t.id)).size === tasks.length, "INVALID_SELECTION");
+    tasks.forEach((task) => campaignAssert(validateAgentSuccessTask(task).ok, "INVALID_SELECTION"));
+    campaignAssert(canonicalCampaignJson(treatments) === canonicalCampaignJson(AGENT_SUCCESS_RATE_TREATMENT_IDS), "FIXED_TREATMENTS_REQUIRED");
+    campaignAssert(limit === "one-case" || limit === "all-selected-cases", "INVALID_EXECUTION_LIMIT");
+    return tasks.slice(0, limit === "one-case" ? 1 : tasks.length).flatMap((task, caseIndex) => AGENT_SUCCESS_RATE_TREATMENT_IDS.map((treatmentId, treatmentIndex) => ({ task, treatmentId, checkpointIndex: caseIndex * 2 + treatmentIndex })));
+}

package/dist/src/experiments/plugins/agentSuccessRate/campaignRecovery.js

@@ -0,0 +1,256 @@
+import { campaignAssert, campaignExact, campaignHash, campaignIdentifier, canonicalCampaignJson, isCampaignDigest, validateCampaignIdentity } from "./campaignIdentity.js";
+import { CAMPAIGN_CHECKPOINT_MAX_BYTES, createCampaignCheckpoint, projectCampaignTreatmentEvidence, requiredCampaignArtifacts, validateCampaignCompletion, validateCampaignRelativePath } from "./campaignCheckpoint.js";
+import { analyzeAgentSuccessTreatment } from "./analysis.js";
+import { classifyAgentSuccessAttempt } from "./repairPolicy.js";
+export const CAMPAIGN_CHECKPOINT_SCHEMA_V2 = "my-dev-kit-lab-agent-success-rate-campaign-checkpoint-v2";
+export const CAMPAIGN_EPOCH_SCHEMA = "campaign-invocation-epoch-v1";
+const equal = (a, b) => canonicalCampaignJson(a) === canonicalCampaignJson(b);
+function timestamp(value) {
+    campaignAssert(typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value) && new Date(value).toISOString() === value, "INVALID_TIMESTAMP");
+}
+function generation(value) { campaignAssert(Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 1_000_000, "INVALID_GENERATION"); }
+export function campaignEpochNamespace(caseId, treatmentId, ordinal, id) {
+    campaignAssert(campaignIdentifier(caseId) && ["raw-full-file", "context-pack"].includes(treatmentId) && [1, 2].includes(ordinal) && /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(id), "INVALID_EPOCH_ID");
+    // Compact, deterministic case/treatment labels leave room for Git's internal paths on Windows.
+    // The full epoch UUID is retained (without separators); the marker also binds the full case identity.
+    const relative = `campaign-epochs-v2/${campaignHash(caseId).slice(0, 12)}/${treatmentId === "raw-full-file" ? "raw" : "pack"}/${ordinal}-${id.replace(/-/g, "")}`;
+    validateCampaignRelativePath(relative);
+    return relative;
+}
+/** Only artifact references change. Source/check/protected paths and scientific evidence remain untouched. */
+export function mapCampaignEpochEvidence(source, namespace, direction) {
+    validateCampaignRelativePath(namespace);
+    const evidence = structuredClone(source);
+    const map = (value) => {
+        if (value === null)
+            return null;
+        validateCampaignRelativePath(value);
+        if (direction === "prefix")
+            return `${namespace}/${value}`;
+        campaignAssert(value.startsWith(`${namespace}/`), "EPOCH_ARTIFACT_SCOPE");
+        const relative = value.slice(namespace.length + 1);
+        validateCampaignRelativePath(relative);
+        return relative;
+    };
+    const apply = (e) => {
+        e.proposedPatchPath = map(e.proposedPatchPath);
+        e.appliedPatchPath = map(e.appliedPatchPath);
+        if (e.realAgent) {
+            e.realAgent.context.contextArtifactPath = map(e.realAgent.context.contextArtifactPath);
+            e.realAgent.agentArtifactDirectory = map(e.realAgent.agentArtifactDirectory);
+            for (const key of Object.keys(e.realAgent.agentArtifacts))
+                e.realAgent.agentArtifacts[key] = map(e.realAgent.agentArtifacts[key]);
+        }
+    };
+    apply(evidence);
+    evidence.attempts?.forEach(a => apply(a.evidence));
+    return evidence;
+}
+export function validateCampaignEpochArtifacts(value, expected) {
+    campaignAssert(Array.isArray(value) && value.length <= 32, "ARTIFACT_REFERENCE_MISMATCH");
+    value.forEach(r => { campaignExact(r, ["path", "kind", "sha256", "bytes"]); validateCampaignRelativePath(r.path); campaignAssert(["patch", "context", "provider"].includes(String(r.kind)) && isCampaignDigest(r.sha256) && Number.isSafeInteger(r.bytes) && Number(r.bytes) >= 0 && Number(r.bytes) <= 32 * 1024 * 1024); });
+    campaignAssert(equal(value.map(({ path, kind }) => ({ path, kind })), expected), "ARTIFACT_REFERENCE_MISMATCH");
+}
+export function validateCampaignLimitAuthorization(value, cp, unit) {
+    campaignExact(value, ["identityDigest", "generation", "revision", "caseId", "treatmentId", "acknowledgeInvocation", "acknowledgeExternalLimit"]);
+    campaignAssert(value.identityDigest === cp.identity.digest && value.generation === cp.generation && value.revision === cp.integrity && value.caseId === unit.caseId && value.treatmentId === unit.treatmentId && value.acknowledgeInvocation === true && value.acknowledgeExternalLimit === true, "LIMIT_AUTHORIZATION_MISMATCH");
+    const previous = unit.epochs.at(-1);
+    campaignAssert(unit.state === "paused-limit" && unit.epochs.length === 1 && previous?.phase === "terminal" && previous.terminal?.reason === "provider-limit" && unit.completion === null, "LIMIT_CONTINUATION_NOT_ELIGIBLE");
+}
+export function sealCampaignCheckpointV2(body) { return { ...body, integrity: campaignHash(canonicalCampaignJson(body)) }; }
+export function createCampaignCheckpointV2(identity, tasks, runId, at) {
+    const old = createCampaignCheckpoint(identity, tasks, at);
+    const cp = sealCampaignCheckpointV2({ schemaVersion: CAMPAIGN_CHECKPOINT_SCHEMA_V2, identity: structuredClone(identity), runId, generation: 1, parentIntegrity: null, createdAt: at, updatedAt: at, units: old.units.map(u => ({ caseId: u.caseId, treatmentId: u.treatmentId, state: "pending", epochs: [], completion: null })), finalization: null });
+    validateCampaignCheckpointV2(cp, tasks);
+    return cp;
+}
+export function validateCampaignCheckpointV2(value, tasks) {
+    campaignAssert(Buffer.byteLength(canonicalCampaignJson(value)) <= CAMPAIGN_CHECKPOINT_MAX_BYTES, "DATA_BOUND");
+    campaignExact(value, ["schemaVersion", "identity", "runId", "generation", "parentIntegrity", "createdAt", "updatedAt", "units", "finalization", "integrity"]);
+    campaignAssert(value.schemaVersion === CAMPAIGN_CHECKPOINT_SCHEMA_V2, "UNSUPPORTED_SCHEMA");
+    validateCampaignIdentity(value.identity);
+    generation(value.generation);
+    campaignAssert(campaignIdentifier(value.runId));
+    timestamp(value.createdAt);
+    timestamp(value.updatedAt);
+    campaignAssert(value.updatedAt >= value.createdAt && (value.parentIntegrity === null || isCampaignDigest(value.parentIntegrity)));
+    const identity = value.identity;
+    const createdAt = value.createdAt;
+    const updatedAt = value.updatedAt;
+    campaignAssert(tasks.length === identity.selected.length && tasks.every((t, i) => t.id === identity.selected[i].caseId && t.benchmarkProject === identity.selected[i].projectId && campaignHash(canonicalCampaignJson(t)) === identity.selected[i].taskDigest), "TASK_IDENTITY_MISMATCH");
+    campaignAssert(Array.isArray(value.units) && value.units.length === tasks.length * 2);
+    const ids = new Set();
+    value.units.forEach((u, index) => {
+        campaignExact(u, ["caseId", "treatmentId", "state", "epochs", "completion"]);
+        const task = tasks[Math.floor(index / 2)];
+        campaignAssert(u.caseId === task.id && u.treatmentId === identity.treatmentOrder[index % 2] && Array.isArray(u.epochs) && u.epochs.length <= 2, "UNIT_IDENTITY");
+        campaignAssert(["pending", "in-progress", "completed", "paused-limit", "incomplete", "uncertain"].includes(String(u.state)));
+        const epochs = u.epochs;
+        epochs.forEach((e, i) => {
+            campaignExact(e, ["schemaVersion", "caseId", "treatmentId", "ordinal", "id", "previousEpochId", "parentRevision", "phase", "configuration", "namespace", "authorization", "reservedAt", "authorizedAt", "terminalAt", "terminal"]);
+            campaignAssert(e.schemaVersion === CAMPAIGN_EPOCH_SCHEMA && e.caseId === u.caseId && e.treatmentId === u.treatmentId && e.ordinal === i + 1 && !ids.has(String(e.id)), "EPOCH_IDENTITY");
+            ids.add(String(e.id));
+            campaignAssert(e.namespace === campaignEpochNamespace(String(u.caseId), String(u.treatmentId), i + 1, String(e.id)) && isCampaignDigest(e.parentRevision) && equal(e.configuration, identity.config), "EPOCH_CONFIGURATION");
+            timestamp(e.reservedAt);
+            campaignAssert(e.reservedAt >= createdAt && e.reservedAt <= updatedAt);
+            campaignExact(e.authorization, ["intent", "scope"]);
+            if (i === 0)
+                campaignAssert(e.previousEpochId === null && e.authorization.intent === "initial" && e.authorization.scope === null);
+            else {
+                const prior = epochs[i - 1];
+                campaignAssert(e.previousEpochId === prior.id && prior.phase === "terminal" && prior.terminal?.reason === "provider-limit" && e.authorization.intent === "limit-continuation", "EPOCH_CONTINUATION");
+                const scope = e.authorization.scope;
+                campaignExact(scope, ["identityDigest", "generation", "revision", "caseId", "treatmentId", "acknowledgeInvocation", "acknowledgeExternalLimit"]);
+                generation(scope.generation);
+                campaignAssert(scope.identityDigest === identity.digest && scope.revision === e.parentRevision && scope.caseId === u.caseId && scope.treatmentId === u.treatmentId && scope.acknowledgeInvocation === true && scope.acknowledgeExternalLimit === true && Number(scope.generation) < Number(value.generation));
+            }
+            campaignAssert(["reserved", "invocation-authorized", "terminal", "uncertain"].includes(String(e.phase)));
+            if (e.phase === "reserved")
+                campaignAssert(e.authorizedAt === null && e.terminalAt === null && e.terminal === null);
+            else {
+                timestamp(e.authorizedAt);
+                campaignAssert(e.authorizedAt >= e.reservedAt && e.authorizedAt <= updatedAt);
+            }
+            if (e.phase !== "terminal")
+                campaignAssert(e.terminalAt === null && e.terminal === null);
+            else {
+                timestamp(e.terminalAt);
+                campaignAssert(e.terminalAt >= String(e.authorizedAt) && e.terminalAt <= updatedAt);
+                const t = e.terminal;
+                campaignExact(t, ["outcome", "reason", "evidence", "artifacts"]);
+                campaignAssert(["evaluated", "provider-limit", "incomplete"].includes(String(t.reason)));
+                const evidence = t.evidence;
+                campaignAssert(equal(evidence, projectCampaignTreatmentEvidence(evidence)), "PRIVATE_OR_INVALID_EVIDENCE");
+                const local = mapCampaignEpochEvidence(evidence, String(e.namespace), "strip");
+                campaignAssert(local.treatmentId === u.treatmentId && t.outcome === (local.realAgent?.providerStatus ?? "not-invoked"), "TERMINAL_OUTCOME");
+                const rows = local.attempts?.map(a => a.evidence) ?? [local];
+                campaignAssert(rows.length > 0 && rows.length <= 1 + identity.config.repairAttempts, "ATTEMPT_BOUND");
+                rows.forEach((a, j) => campaignAssert(a.treatmentId === u.treatmentId && (!a.realAgent || (a.realAgent.providerId === identity.config.providerId && a.realAgent.attempt === j + 1)) && (!local.attempts || local.attempts[j].attemptNumber === j + 1), "ATTEMPT_IDENTITY"));
+                if (local.attempts) {
+                    const { attempts: _attempts, ...final } = local;
+                    campaignAssert(equal(final, rows.at(-1)), "FINAL_ATTEMPT_MISMATCH");
+                    local.attempts.forEach((a, j) => {
+                        const row = rows[j];
+                        const classification = classifyAgentSuccessAttempt(row, analyzeAgentSuccessTreatment(task, row).metrics);
+                        campaignAssert(a.providerId === identity.config.providerId && a.providerStatus === row.realAgent?.providerStatus && a.failureCategory === classification.category && a.repairEligible === classification.repairEligible && a.taskSuccess === classification.taskSuccess && equal(a.cleanupResult, row.cleanup) && equal(a.tokenUsage, row.agentTokenUsage) && a.durationMs === row.timing.agentDurationMs && a.completedAt >= a.startedAt, "ATTEMPT_PROOF_MISMATCH");
+                        if (j > 0)
+                            campaignAssert(local.attempts[j - 1].repairEligible && local.attempts[j - 1].taskSuccess === false, "REPAIR_NOT_ELIGIBLE");
+                    });
+                }
+                campaignAssert(t.reason === "provider-limit" || t.outcome !== "agent-limit-reached", "LIMIT_CLASSIFICATION_REQUIRED");
+                const expected = requiredCampaignArtifacts(local, task, t.reason !== "evaluated").map(r => ({ ...r, path: `${e.namespace}/${r.path}` }));
+                validateCampaignEpochArtifacts(t.artifacts, expected);
+                if (t.reason === "provider-limit")
+                    campaignAssert(t.outcome === "agent-limit-reached" && local.realAgent?.providerInvoked === true && local.availability !== "complete" && !local.patch.attempted && local.cleanup.attempted && local.cleanup.removed && local.realAgent.cwdCleanup.attempted && local.realAgent.cwdCleanup.removed && local.protectedIntegrity.status === "intact", "LIMIT_TERMINAL_PROOF");
+                if (t.reason === "evaluated") {
+                    const completion = { sourceSchema: "my-dev-kit-lab-agent-success-rate-execution-v1", evidence: local, taskSuccess: analyzeAgentSuccessTreatment(task, local).metrics.taskSuccess.value === true, artifacts: t.artifacts.map(r => ({ ...r, path: r.path.slice(String(e.namespace).length + 1) })) };
+                    validateCampaignCompletion(completion, identity, task, String(u.treatmentId));
+                }
+            }
+        });
+        const last = u.epochs.at(-1);
+        if (u.state === "pending")
+            campaignAssert(last === undefined && u.completion === null);
+        else {
+            campaignAssert(last !== undefined);
+            if (u.state === "in-progress")
+                campaignAssert(["reserved", "invocation-authorized"].includes(last.phase));
+            if (u.state === "uncertain")
+                campaignAssert(last.phase === "uncertain");
+            if (["completed", "paused-limit", "incomplete"].includes(String(u.state)))
+                campaignAssert(last.phase === "terminal" && last.terminal?.reason === (u.state === "completed" ? "evaluated" : u.state === "paused-limit" ? "provider-limit" : "incomplete"));
+            if (u.state === "completed")
+                campaignAssert(equal(u.completion, { sourceSchema: "my-dev-kit-lab-agent-success-rate-execution-v1", evidence: last.terminal.evidence, taskSuccess: analyzeAgentSuccessTreatment(task, last.terminal.evidence).metrics.taskSuccess.value === true, artifacts: last.terminal.artifacts }));
+            else
+                campaignAssert(u.completion === null);
+        }
+    });
+    if (value.finalization !== null) {
+        campaignExact(value.finalization, ["completedAt", "artifacts"]);
+        timestamp(value.finalization.completedAt);
+        campaignAssert(value.units.every(u => u.state === "completed") && value.finalization.completedAt <= value.updatedAt);
+        validateCampaignEpochArtifacts(value.finalization.artifacts, [{ path: "agent-success-rate-analysis.json", kind: "provider" }, { path: "agent-success-rate-execution.json", kind: "provider" }]);
+    }
+    const { integrity, ...body } = value;
+    campaignAssert(isCampaignDigest(integrity) && integrity === campaignHash(canonicalCampaignJson(body)), "CHECKPOINT_INTEGRITY");
+}
+export function transitionCampaignCheckpointV2(previous, tasks, action, at) {
+    validateCampaignCheckpointV2(previous, tasks);
+    campaignAssert(previous.finalization === null, "COMPLETED_IMMUTABLE");
+    const { integrity, ...body } = structuredClone(previous);
+    body.generation++;
+    body.parentIntegrity = integrity;
+    body.updatedAt = at;
+    if (action.kind === "finalize") {
+        campaignAssert(body.units.every(u => u.state === "completed"), "INCOMPLETE_CAMPAIGN");
+        body.finalization = structuredClone(action.finalization);
+    }
+    else {
+        campaignAssert(Number.isSafeInteger(action.index) && action.index >= 0 && action.index < body.units.length);
+        campaignAssert(body.units.slice(0, action.index).every(u => u.state === "completed"), "QUEUE_ORDER");
+        const u = body.units[action.index];
+        const last = u.epochs.at(-1);
+        campaignAssert(u.state !== "completed", "COMPLETED_IMMUTABLE");
+        if (action.kind === "reserve") {
+            if (action.authorization)
+                validateCampaignLimitAuthorization(action.authorization, previous, previous.units[action.index]);
+            else
+                campaignAssert(u.state === "pending" && u.epochs.length === 0, "UNIT_NOT_PENDING");
+            const ordinal = (u.epochs.length + 1);
+            const e = { schemaVersion: CAMPAIGN_EPOCH_SCHEMA, caseId: u.caseId, treatmentId: u.treatmentId, ordinal, id: action.epochId, previousEpochId: last?.id ?? null, parentRevision: integrity, phase: "reserved", configuration: structuredClone(previous.identity.config), namespace: campaignEpochNamespace(u.caseId, u.treatmentId, ordinal, action.epochId), authorization: action.authorization ? { intent: "limit-continuation", scope: structuredClone(action.authorization) } : { intent: "initial", scope: null }, reservedAt: at, authorizedAt: null, terminalAt: null, terminal: null };
+            u.epochs.push(e);
+            u.state = "in-progress";
+        }
+        else {
+            campaignAssert(last && u.state === "in-progress", "INVALID_EPOCH_TRANSITION");
+            if (action.kind === "authorize") {
+                campaignAssert(last.phase === "reserved");
+                last.phase = "invocation-authorized";
+                last.authorizedAt = at;
+            }
+            else {
+                campaignAssert(last.phase === "invocation-authorized");
+                if (action.kind === "uncertain") {
+                    last.phase = "uncertain";
+                    u.state = "uncertain";
+                }
+                else {
+                    last.phase = "terminal";
+                    last.terminalAt = at;
+                    last.terminal = structuredClone(action.terminal);
+                    u.state = action.terminal.reason === "evaluated" ? "completed" : action.terminal.reason === "provider-limit" ? "paused-limit" : "incomplete";
+                    if (u.state === "completed")
+                        u.completion = { sourceSchema: "my-dev-kit-lab-agent-success-rate-execution-v1", evidence: structuredClone(action.terminal.evidence), taskSuccess: analyzeAgentSuccessTreatment(tasks[Math.floor(action.index / 2)], action.terminal.evidence).metrics.taskSuccess.value === true, artifacts: structuredClone(action.terminal.artifacts) };
+                }
+            }
+        }
+    }
+    const result = sealCampaignCheckpointV2(body);
+    validateCampaignCheckpointV2(result, tasks);
+    return result;
+}
+/** Reconstruct the exact permitted transition; no last-writer-wins or arbitrary unit replacement. */
+export function validateCampaignHistoryV2(previous, next, tasks) {
+    let action;
+    if (previous.finalization === null && next.finalization !== null)
+        action = { kind: "finalize", finalization: next.finalization };
+    else {
+        const changes = next.units.map((u, i) => equal(u, previous.units[i]) ? -1 : i).filter(i => i >= 0);
+        campaignAssert(changes.length === 1, "INVALID_HISTORY");
+        const index = changes[0];
+        const a = previous.units[index], b = next.units[index];
+        const e = b.epochs.at(-1);
+        campaignAssert(e !== undefined, "INVALID_HISTORY");
+        if (b.epochs.length === a.epochs.length + 1)
+            action = { kind: "reserve", index, epochId: e.id, ...(e.authorization.intent === "limit-continuation" ? { authorization: e.authorization.scope } : {}) };
+        else if (e.phase === "invocation-authorized")
+            action = { kind: "authorize", index };
+        else if (e.phase === "uncertain")
+            action = { kind: "uncertain", index };
+        else {
+            campaignAssert(e.phase === "terminal" && e.terminal !== null, "INVALID_HISTORY");
+            action = { kind: "terminal", index, terminal: e.terminal };
+        }
+    }
+    campaignAssert(equal(transitionCampaignCheckpointV2(previous, tasks, action, next.updatedAt), next), "INVALID_HISTORY");
+}

package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js

@@ -0,0 +1,207 @@
+import { rmdir } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { runAgentPrompt } from "../../../agents/runAgentPrompt.js";
+import { campaignAssert, canonicalCampaignJson, compareCampaignIdentity } from "./campaignIdentity.js";
+import { buildCampaignQueue } from "./campaignQueue.js";
+import { createCampaignCheckpointV2, transitionCampaignCheckpointV2, validateCampaignLimitAuthorization } from "./campaignRecovery.js";
+import { acquireCampaignDispatch, allocateCampaignEpoch, assertCampaignDispatch, persistCampaignSupportingArtifacts, prepareCampaignEpochTerminal, prepareCampaignProviderOutput, publishCampaignFinalArtifacts, readCampaignCheckpoint, readCampaignCheckpointV2, releaseCampaignDispatch, writeCampaignCheckpointV2 } from "./campaignCheckpointStore.js";
+import { validatePreparedCampaign } from "./campaignExecution.js";
+import { boundMessage, executeAgentSuccessTreatment, resolveControlledBenchmarkProject, resolveSandboxRuntimeRoot } from "./execution.js";
+import { executeAgentSuccessRepairTreatment } from "./repairExecution.js";
+import { agentSuccessAgentArtifactDirectory, createRealAgentProposalSource } from "./realAgentExecution.js";
+import { defaultContextDependencies, sealTask } from "./plugin.js";
+import { projectAgentFacingTask } from "./agentTaskProjection.js";
+import { AgentSuccessPackContextBuilder, buildContextTaskInput, buildRawFullFileContext, DEFAULT_AGENT_SUCCESS_KIT_COMMAND } from "./contextGeneration.js";
+import { analyzeAgentSuccessRate } from "./analysis.js";
+import { buildAgentSuccessRateExecutionArtifact, validateAgentSuccessRateExecutionArtifact, AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { buildAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateArtifactFamily, AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE } from "./analysisArtifact.js";
+function blocked(outcome, unit) {
+    const cp = outcome.checkpoint;
+    const epoch = unit.epochs.at(-1);
+    outcome.status = unit.state === "paused-limit" ? "provider-limit" : "recovery-required";
+    outcome.blockedUnit = { caseId: unit.caseId, treatmentId: unit.treatmentId, generation: cp.generation, revision: cp.integrity, phase: epoch?.phase ?? "pending", reason: unit.state === "paused-limit" ? "provider-limit" : epoch?.phase === "reserved" ? "reserved-not-replayed" : unit.state === "incomplete" ? "terminal-incomplete" : "potentially-invoked" };
+}
+export async function executeResumableCampaign(prepared, args) {
+    const outcome = { status: "failed", checkpoint: null, errors: [], scientificArtifactsWritten: false, blockedUnit: null };
+    let owner;
+    let verified;
+    let packBuilder;
+    let activeRuntime;
+    const at = () => new Date().toISOString();
+    try {
+        campaignAssert(args.intent && ["new-resumable", "resume", "authorize-limit-continuation"].includes(args.intent.kind), "EXPLICIT_INTENT_REQUIRED");
+        // The closed runtime intent also rejects force-replay or a generic resume boolean.
+        campaignAssert(Object.keys(args.intent).sort().join(",") === (args.intent.kind === "authorize-limit-continuation" ? "authorization,kind" : "kind"), "INVALID_INTENT");
+        verified = await validatePreparedCampaign(prepared);
+        const { input, identity } = verified;
+        const { boundary, tasks, provider } = input;
+        if (provider.kind === "real-agent")
+            campaignAssert(provider.includeRealAgents === true && provider.trustedExecutable && provider.trustedExecutable.providerId === provider.providerId, "REAL_PROVIDER_AUTHORIZATION_REQUIRED");
+        const queue = buildCampaignQueue(tasks, input.treatments, "all-selected-cases");
+        campaignAssert(["one-case", "all-selected-cases"].includes(args.limit), "INVALID_EXECUTION_LIMIT");
+        owner = await acquireCampaignDispatch(boundary, identity);
+        const fresh = async () => {
+            const current = await validatePreparedCampaign(prepared);
+            campaignAssert(compareCampaignIdentity(identity, current.identity) === "MATCH", "IDENTITY_CHANGED_BEFORE_DISPATCH");
+            await assertCampaignDispatch(boundary, identity, owner);
+        };
+        const read = async () => {
+            await fresh();
+            const stored = await readCampaignCheckpointV2(boundary, identity, tasks);
+            campaignAssert(stored.identityVerification === "MATCH", "IDENTITY_UNVERIFIABLE");
+            campaignAssert(stored.temporaryFiles.length === 0, "CHECKPOINT_PUBLICATION_UNRESOLVED");
+            return stored;
+        };
+        const publish = async (cp) => {
+            await fresh();
+            await writeCampaignCheckpointV2(boundary, cp, tasks, phase => args.hooks?.publication?.(cp.generation, phase), owner);
+            const stored = await read();
+            campaignAssert(stored.checkpoint?.integrity === cp.integrity, "CLAIM_READBACK_FAILED");
+            outcome.checkpoint = stored.checkpoint;
+        };
+        const stored = await read();
+        if (args.intent.kind === "new-resumable") {
+            campaignAssert(stored.checkpoint === null, "CAMPAIGN_ALREADY_EXISTS");
+            // No copy or migration, even when a historical chain happens to share the identity.
+            const legacy = await readCampaignCheckpoint(boundary, identity, tasks);
+            campaignAssert(legacy.checkpoint === null && legacy.temporaryFiles.length === 0, "LEGACY_CAMPAIGN_READ_ONLY");
+            await publish(createCampaignCheckpointV2(identity, tasks, args.runId, at()));
+        }
+        else {
+            campaignAssert(stored.reusable && stored.checkpoint !== null, "RESUMABLE_CAMPAIGN_REQUIRED_NO_V1_MIGRATION");
+            outcome.checkpoint = stored.checkpoint;
+        }
+        const alreadyComplete = outcome.checkpoint.units.every(u => u.state === "completed");
+        // Validate a continuation request before accepting any complete/blocked/no-work shortcut.
+        let authorizedIndex;
+        if (args.intent.kind === "authorize-limit-continuation") {
+            authorizedIndex = outcome.checkpoint.units.findIndex(u => u.state !== "completed");
+            campaignAssert(authorizedIndex >= 0, "COMPLETED_IMMUTABLE");
+            validateCampaignLimitAuthorization(args.intent.authorization, outcome.checkpoint, outcome.checkpoint.units[authorizedIndex]);
+        }
+        if (!alreadyComplete && provider.kind === "trusted-simulated-provider")
+            packBuilder = new AgentSuccessPackContextBuilder("trusted-simulated-kit", provider.contextDependencies);
+        if (!alreadyComplete && provider.kind === "real-agent")
+            packBuilder = new AgentSuccessPackContextBuilder(provider.kitCommand ?? DEFAULT_AGENT_SUCCESS_KIT_COMMAND, provider.contextDependencies ?? defaultContextDependencies());
+        let caseProcessed;
+        for (const unit of queue) {
+            let state = outcome.checkpoint.units[unit.checkpointIndex];
+            if (state.state === "completed")
+                continue;
+            if (args.limit === "one-case" && caseProcessed !== undefined && caseProcessed !== unit.task.id)
+                break;
+            if (state.state !== "pending" && authorizedIndex !== unit.checkpointIndex) {
+                blocked(outcome, state);
+                return outcome;
+            }
+            caseProcessed = unit.task.id;
+            // Read and verify the whole chain again before reserving. No local snapshot alone authorizes dispatch.
+            const latest = (await read()).checkpoint;
+            campaignAssert(latest.integrity === outcome.checkpoint.integrity, "STALE_REVISION");
+            const reserve = transitionCampaignCheckpointV2(latest, tasks, { kind: "reserve", index: unit.checkpointIndex, epochId: randomUUID(), ...(authorizedIndex === unit.checkpointIndex && args.intent.kind === "authorize-limit-continuation" ? { authorization: args.intent.authorization } : {}) }, at());
+            const epoch = reserve.units[unit.checkpointIndex].epochs.at(-1);
+            const epochRoot = await allocateCampaignEpoch(boundary, identity, owner, epoch);
+            await publish(reserve);
+            // Context preparation occurs before provider authorization; a crash here does not create an implicit replay policy.
+            const forbidden = new Map();
+            const executionTask = provider.kind === "deterministic-fixture" ? unit.task : sealTask(unit.task, forbidden);
+            const canonicalProjectRoot = await resolveControlledBenchmarkProject(boundary.toolRoot, unit.task.benchmarkProject);
+            const contextInput = buildContextTaskInput(executionTask);
+            const contexts = packBuilder ? { "raw-full-file": await buildRawFullFileContext(contextInput, canonicalProjectRoot), "context-pack": await packBuilder.build(contextInput, canonicalProjectRoot) } : null;
+            activeRuntime = await resolveSandboxRuntimeRoot(epochRoot, boundary.toolRoot);
+            await publish(transitionCampaignCheckpointV2(outcome.checkpoint, tasks, { kind: "authorize", index: unit.checkpointIndex }, at()));
+            await fresh();
+            const claimed = (await read()).checkpoint;
+            state = claimed.units[unit.checkpointIndex];
+            campaignAssert(claimed.integrity === outcome.checkpoint.integrity && state.epochs.at(-1).id === epoch.id && state.epochs.at(-1).phase === "invocation-authorized", "INVOCATION_NOT_AUTHORIZED");
+            const common = { task: executionTask, treatmentId: unit.treatmentId, runId: `${outcome.checkpoint.runId}-${epoch.id}`, canonicalProjectRoot, runtimeRoot: activeRuntime, privateRoots: [boundary.outputRoot, boundary.toolRoot], dependencies: input.dependencies };
+            // Explicit fixture branch only. Missing provider context never selects a fixture patch.
+            campaignAssert(provider.kind === "deterministic-fixture" || contexts !== null, "PROVIDER_CONTEXT_REQUIRED");
+            const result = provider.kind !== "deterministic-fixture" && contexts ? await executeAgentSuccessRepairTreatment({ ...common, repair: {
+                    providerId: provider.providerId, repairAttempts: provider.repairAttempts,
+                    createProposalSource: ({ attemptNumber, feedback }) => createRealAgentProposalSource({ settings: { providerId: provider.providerId, timeoutMs: provider.timeoutMs, ...(provider.kind === "real-agent" ? { trustedExecutable: provider.trustedExecutable, env: provider.env } : {}), outDir: epochRoot, privateRoots: common.privateRoots, protectedRoots: [boundary.toolRoot, boundary.outputRoot, activeRuntime, canonicalProjectRoot], runAgent: async (request) => {
+                                await fresh();
+                                campaignAssert((await read()).checkpoint?.integrity === claimed.integrity, "STALE_INVOCATION_CLAIM");
+                                await prepareCampaignProviderOutput(boundary, identity, owner, `${epoch.namespace}/${agentSuccessAgentArtifactDirectory(unit.task.benchmarkProject, unit.task.id, unit.treatmentId, attemptNumber)}`);
+                                return provider.kind === "real-agent" ? runAgentPrompt(request) : provider.runAgent(request);
+                            } }, runId: common.runId, task: projectAgentFacingTask(executionTask), contexts, forbidden: forbidden.get(unit.task.id) ?? [], attemptNumber, ...(feedback ? { repair: { feedback, maxAttempts: 1 + provider.repairAttempts } } : {}) })
+                } }) : await executeAgentSuccessTreatment(common);
+            await rmdir(activeRuntime);
+            activeRuntime = undefined;
+            await fresh();
+            await persistCampaignSupportingArtifacts(boundary, identity, owner, unit.task, result, epoch);
+            const reason = result.evidence.realAgent?.providerStatus === "agent-limit-reached" ? "provider-limit" : result.evidence.status === "completed" && result.evidence.availability === "complete" ? "evaluated" : "incomplete";
+            const terminal = await prepareCampaignEpochTerminal(boundary, identity, unit.task, epoch, result.evidence, reason);
+            await publish(transitionCampaignCheckpointV2(outcome.checkpoint, tasks, { kind: "terminal", index: unit.checkpointIndex, terminal }, at()));
+            authorizedIndex = undefined;
+            if (reason !== "evaluated") {
+                blocked(outcome, outcome.checkpoint.units[unit.checkpointIndex]);
+                return outcome;
+            }
+        }
+        const complete = outcome.checkpoint.units.every(u => u.state === "completed");
+        if (!complete) {
+            outcome.status = "partial";
+            return outcome;
+        }
+        const cp = (await read()).checkpoint;
+        outcome.checkpoint = cp;
+        {
+            const cases = tasks.map((task, i) => ({ caseId: task.id, caseName: task.title, benchmarkProject: task.benchmarkProject, taskLocality: task.taskLocality, fixtureId: provider.kind === "deterministic-fixture" ? task.deterministicFixture.id : "", treatments: [cp.units[i * 2].completion.evidence, cp.units[i * 2 + 1].completion.evidence] }));
+            const analysis = analyzeAgentSuccessRate(tasks, cases, provider.kind === "deterministic-fixture" ? "deterministic-fixture" : "real-agent");
+            // Stable final metadata also permits safe recovery after a partial final artifact publication.
+            const completedAt = cp.finalization?.completedAt ?? cp.updatedAt;
+            const common = { runId: cp.runId, pluginId: "agent-success-rate", pluginSchemaVersion: "1.0.0", startedAt: cp.createdAt, completedAt };
+            const execution = buildAgentSuccessRateExecutionArtifact({ ...common, cases, ...(provider.kind === "deterministic-fixture" ? {} : { realAgent: { providerId: provider.providerId, timeoutMs: provider.timeoutMs, repairAttempts: provider.repairAttempts, attemptsPerTreatment: (1 + provider.repairAttempts), promptTransport: "stdin", contextEffectEvaluated: analysis.contextEffectEvaluated } }) });
+            const scientific = buildAgentSuccessRateAnalysisArtifact({ ...common, analysis });
+            campaignAssert([...validateAgentSuccessRateExecutionArtifact(execution), ...validateAgentSuccessRateAnalysisArtifact(scientific), ...validateAgentSuccessRateArtifactFamily(execution, scientific)].length === 0, "SCIENTIFIC_ARTIFACT_INVALID");
+            const finalization = await publishCampaignFinalArtifacts(boundary, identity, owner, [{ relativePath: AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE, content: `${JSON.stringify(execution, null, 2)}\n` }, { relativePath: AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE, content: `${JSON.stringify(scientific, null, 2)}\n` }], cp.runId, completedAt);
+            if (cp.finalization === null)
+                await publish(transitionCampaignCheckpointV2(cp, tasks, { kind: "finalize", finalization }, at()));
+            else
+                campaignAssert(canonicalCampaignJson(finalization) === canonicalCampaignJson(cp.finalization), "FINAL_ARTIFACT_INCONSISTENT");
+        }
+        outcome.scientificArtifactsWritten = true;
+        outcome.status = alreadyComplete ? "already-complete" : "completed";
+    }
+    catch (error) {
+        outcome.status = "failed";
+        outcome.errors.push(boundMessage(error instanceof Error ? error.message : "CAMPAIGN_FAILED", verified ? [verified.input.boundary.outputRoot, verified.input.boundary.toolRoot, verified.input.sourceRoot ?? ""] : []));
+        if (owner && verified)
+            try {
+                outcome.checkpoint = (await readCampaignCheckpointV2(verified.input.boundary, verified.identity, verified.input.tasks)).checkpoint;
+            }
+            catch {
+                outcome.errors.push("CHECKPOINT_READ_FAILED");
+            }
+    }
+    finally {
+        if (packBuilder)
+            try {
+                campaignAssert((await packBuilder.dispose()) === null, "CONTEXT_CLEANUP_FAILED");
+            }
+            catch {
+                outcome.errors.push("CONTEXT_CLEANUP_FAILED");
+                outcome.status = "failed";
+            }
+        if (activeRuntime)
+            try {
+                await rmdir(activeRuntime);
+            }
+            catch (error) {
+                if (error.code !== "ENOENT") {
+                    outcome.errors.push("RUNTIME_ROOT_NOT_REMOVED");
+                    outcome.status = "failed";
+                }
+            }
+        if (owner && verified)
+            try {
+                await releaseCampaignDispatch(verified.input.boundary, verified.identity, owner);
+            }
+            catch {
+                outcome.errors.push("DISPATCH_RELEASE_FAILED");
+                outcome.status = "failed";
+            }
+    }
+    return outcome;
+}

package/dist/src/experiments/plugins/agentSuccessRate/config.js

@@ -0,0 +1,123 @@
+import { parseCommandString } from "../../../core/commandLine.js";
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+import { DEFAULT_AGENT_SUCCESS_KIT_COMMAND } from "./contextGeneration.js";
+import { AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS } from "./realAgentExecution.js";
+export const AGENT_SUCCESS_RATE_REAL_AGENT_IDS = ["codex", "claude"];
+export const AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES = [0, 1, 2];
+export const defaultAgentSuccessRateConfig = {
+    outDir: "lab-output/agent-success-rate"
+};
+const SUPPORTED_FIELDS = ["outDir", "caseIds", "benchmarkProjects", "agentId", "includeRealAgents", "timeoutMs", "kitCommand", "repairAttempts"];
+/** Names that are recognized only to give a precise rejection. */
+const REJECTED_FIELD_MESSAGES = {
+    commandTemplate: "commandTemplate is not supported; real providers use the fixed adapter invocation.",
+    campaignPreset: "campaignPreset is not supported by agent-success-rate.",
+    retryUntilSuccess: "retryUntilSuccess is not supported; repair attempts are bounded by repairAttempts (maximum 2).",
+    unlimitedRepairs: "unlimitedRepairs is not supported; repair attempts are bounded by repairAttempts (maximum 2).",
+    providerMatrix: "providerMatrix is not supported; exactly one provider is selected per run.",
+    resumeSession: "resumeSession is not supported.",
+    resumeFromCheckpoint: "resumeFromCheckpoint is not supported.",
+    automaticProviderSwitching: "automaticProviderSwitching is not supported; a repair always uses the selected provider.",
+    strategies: "strategies is not supported; the two treatments are fixed.",
+    agentIds: "agentIds is not supported; exactly one provider is selected per run.",
+    agents: "agents is not supported; exactly one provider is selected per run."
+};
+export const agentSuccessRateConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "caseIds", type: "array", description: "Task IDs to include; input order is preserved." },
+        { name: "benchmarkProjects", type: "array", description: "Benchmark project IDs to include; input order is preserved." },
+        { name: "agentId", type: "string", description: "Real provider for real-agent mode: codex or claude. Absent selects deterministic-fixture mode." },
+        { name: "includeRealAgents", type: "boolean", description: "Must be exactly true whenever agentId is set; rejected otherwise." },
+        { name: "timeoutMs", type: "number", description: "Per-attempt provider timeout in ms (real-agent mode only)." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for context-pack retrieval (real-agent mode only)." },
+        { name: "repairAttempts", type: "number", description: "Repairs allowed after the initial attempt: 0 (default), 1 or 2. Real-agent mode only; at most 3 attempts per treatment." }
+    ]
+};
+export function validateAgentSuccessRateConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["agent success rate config must be an object."]);
+    }
+    const errors = [];
+    const raw = isPlainObject(config) ? config : {};
+    const unsupported = Object.keys(raw).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    const explained = unsupported.filter((key) => key in REJECTED_FIELD_MESSAGES);
+    const unknown = unsupported.filter((key) => !(key in REJECTED_FIELD_MESSAGES));
+    for (const key of explained.sort())
+        errors.push(REJECTED_FIELD_MESSAGES[key]);
+    if (unknown.length > 0) {
+        errors.push(`Unsupported agent-success-rate config field(s): ${unknown.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultAgentSuccessRateConfig, config);
+    const outDir = normalized.outDir;
+    if (typeof outDir !== "string" || !outDir.trim())
+        errors.push("outDir must be a non-empty string.");
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+        else if (new Set(value).size !== value.length) {
+            errors.push(`${field} must not contain duplicate entries.`);
+        }
+    }
+    // Real-agent fields are validated against the RAW object so defaults can never look like an opt-in.
+    const agentId = raw.agentId;
+    if (agentId === undefined) {
+        if (raw.includeRealAgents === true)
+            errors.push("includeRealAgents requires agentId; real-agent mode is entered only by naming one provider.");
+        else if (raw.includeRealAgents !== undefined && typeof raw.includeRealAgents !== "boolean")
+            errors.push("includeRealAgents must be a boolean.");
+        if (raw.timeoutMs !== undefined)
+            errors.push("timeoutMs is only supported together with agentId.");
+        if (raw.kitCommand !== undefined)
+            errors.push("kitCommand is only supported together with agentId.");
+        if (raw.repairAttempts !== undefined) {
+            if (!isRepairAttemptsValue(raw.repairAttempts))
+                errors.push("repairAttempts must be 0, 1 or 2.");
+            else if (raw.repairAttempts > 0)
+                errors.push("repairAttempts greater than 0 requires real-agent mode (agentId together with includeRealAgents=true); deterministic-fixture mode has no repair.");
+        }
+    }
+    else {
+        if (Array.isArray(agentId)) {
+            errors.push("agentId must name exactly one provider (codex or claude); multiple providers are not supported.");
+        }
+        else if (typeof agentId !== "string" || !AGENT_SUCCESS_RATE_REAL_AGENT_IDS.includes(agentId)) {
+            errors.push(`agentId must be one of: ${AGENT_SUCCESS_RATE_REAL_AGENT_IDS.join(", ")}.`);
+        }
+        if (raw.includeRealAgents !== true)
+            errors.push("agentId requires includeRealAgents to be exactly true.");
+        if (raw.timeoutMs !== undefined) {
+            const timeoutMs = raw.timeoutMs;
+            if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || !Number.isInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS) {
+                errors.push(`timeoutMs must be a finite positive integer no greater than ${AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS}.`);
+            }
+        }
+        if (raw.repairAttempts !== undefined && !isRepairAttemptsValue(raw.repairAttempts))
+            errors.push("repairAttempts must be 0, 1 or 2.");
+        if (raw.kitCommand !== undefined) {
+            const kitCommand = raw.kitCommand;
+            let supported = false;
+            if (typeof kitCommand === "string" && kitCommand.trim()) {
+                try {
+                    supported = parseCommandString(kitCommand).executable.length > 0;
+                }
+                catch {
+                    supported = false;
+                }
+            }
+            if (!supported)
+                errors.push("kitCommand must be a non-empty my-dev-kit command string.");
+        }
+        else {
+            normalized.kitCommand = DEFAULT_AGENT_SUCCESS_KIT_COMMAND;
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}
+function isRepairAttemptsValue(value) {
+    return typeof value === "number" && AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES.includes(value);
+}