@dailephd/my-dev-kit-lab 0.9.0 → 0.9.2

package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js

@@ -0,0 +1,314 @@
+import { lstat, mkdtemp, readFile, realpath, rm, rmdir } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { fileURLToPath } from "node:url";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { trustedProviderObservation } from "../../../agents/providerBuildObservation.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { scanProjectTree } from "../../../evaluation/benchmarkSandbox/treeSnapshot.js";
+import { buildCampaignQueue } from "./campaignQueue.js";
+import { campaignAssert, canonicalCampaignJson, compareCampaignIdentity, createCampaignIdentity } from "./campaignIdentity.js";
+import { createCampaignCheckpoint, transitionCampaignCheckpoint } from "./campaignCheckpoint.js";
+import { acquireCampaignDispatch, assertCampaignDispatch, persistCampaignArtifact, persistCampaignSupportingArtifacts, prepareCampaignCompletion, prepareCampaignProviderOutput, readCampaignCheckpoint, releaseCampaignDispatch, writeCampaignCheckpoint } from "./campaignCheckpointStore.js";
+import { boundMessage, executeAgentSuccessTreatment, resolveControlledBenchmarkProject, resolveSandboxRuntimeRoot } from "./execution.js";
+import { executeAgentSuccessRepairTreatment } from "./repairExecution.js";
+import { agentSuccessAgentArtifactDirectory, createRealAgentProposalSource } from "./realAgentExecution.js";
+import { projectAgentFacingTask } from "./agentTaskProjection.js";
+import { AgentSuccessPackContextBuilder, buildContextTaskInput, buildRawFullFileContext } from "./contextGeneration.js";
+import { analyzeAgentSuccessRate } from "./analysis.js";
+import { buildAgentSuccessRateExecutionArtifact, validateAgentSuccessRateExecutionArtifact, AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { buildAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateArtifactFamily, AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE } from "./analysisArtifact.js";
+import { defaultContextDependencies, sealTask, writeAgentSuccessRateArtifactTargets } from "./plugin.js";
+import { DEFAULT_AGENT_SUCCESS_KIT_COMMAND } from "./contextGeneration.js";
+import { executeResumableCampaign } from "./campaignResumableExecution.js";
+const preparations = new WeakMap();
+async function establishPhysicalRoot(root) {
+    const lexical = path.resolve(root);
+    const stat = await lstat(lexical);
+    campaignAssert(stat.isDirectory() && !stat.isSymbolicLink(), "IDENTITY_INPUT_UNSAFE");
+    return { lexical, physical: await realpath(lexical) };
+}
+async function currentBytes(file, authorizedRoot, descendantsValidated = false) {
+    const lexicalRoot = authorizedRoot.lexical;
+    const lexicalFile = path.resolve(file);
+    const relative = path.relative(lexicalRoot, lexicalFile);
+    campaignAssert(relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative), "IDENTITY_INPUT_UNSAFE");
+    // Canonicalizing the trusted root accepts OS aliases of that root. A file is then
+    // checked against the corresponding child reached from the canonical root itself.
+    let physicalCandidate = authorizedRoot.physical;
+    if (!descendantsValidated) {
+        for (const part of relative.split(path.sep)) {
+            physicalCandidate = path.join(physicalCandidate, part);
+            const componentStat = await lstat(physicalCandidate);
+            campaignAssert(!componentStat.isSymbolicLink(), "IDENTITY_INPUT_UNSAFE");
+        }
+    }
+    else {
+        physicalCandidate = path.join(physicalCandidate, relative);
+    }
+    const authorizedFileStat = await lstat(physicalCandidate);
+    campaignAssert(authorizedFileStat.isFile() && !authorizedFileStat.isSymbolicLink() && authorizedFileStat.size <= 32 * 1024 * 1024, "IDENTITY_INPUT_UNSAFE");
+    const stat = await lstat(lexicalFile);
+    campaignAssert(stat.isFile() && !stat.isSymbolicLink() && stat.size <= 32 * 1024 * 1024, "IDENTITY_INPUT_UNSAFE");
+    campaignAssert(authorizedFileStat?.isFile() && authorizedFileStat.dev === stat.dev && authorizedFileStat.ino === stat.ino, "IDENTITY_INPUT_UNSAFE");
+    const bytes = await readFile(lexicalFile);
+    const after = await lstat(lexicalFile);
+    const authorizedAfter = await lstat(physicalCandidate);
+    campaignAssert(bytes.length === stat.size && after.isFile() && !after.isSymbolicLink() && after.size === stat.size && after.dev === stat.dev && after.ino === stat.ino && after.mtimeMs === stat.mtimeMs && authorizedAfter.dev === authorizedFileStat.dev && authorizedAfter.ino === authorizedFileStat.ino && authorizedAfter.size === authorizedFileStat.size && authorizedAfter.mtimeMs === authorizedFileStat.mtimeMs, "IDENTITY_INPUT_CHANGED");
+    return bytes;
+}
+async function produceIdentity(input) {
+    buildCampaignQueue(input.tasks, input.treatments, "all-selected-cases");
+    const sourceRoot = input.sourceRoot ?? fileURLToPath(new URL("../../../../", import.meta.url));
+    const physicalSourceRoot = await establishPhysicalRoot(sourceRoot);
+    const sourceDirectory = path.join(sourceRoot, "src");
+    const sourceDirectoryStat = await lstat(sourceDirectory);
+    campaignAssert(sourceDirectoryStat.isDirectory() && !sourceDirectoryStat.isSymbolicLink(), "IDENTITY_INPUT_UNSAFE");
+    const sources = await scanProjectTree(sourceDirectory, [".git", "node_modules"]);
+    const physicalCorpusRoot = input.corpusFile === null ? null : await establishPhysicalRoot(input.corpusRoot);
+    campaignAssert(sources.length <= 9998, "DATA_BOUND");
+    const sourceFiles = [];
+    for (const file of [...sources.map((p) => `src/${p}`), "package.json", "package-lock.json"])
+        sourceFiles.push({ path: file, bytes: await currentBytes(resolveWithinRoot(sourceRoot, file), physicalSourceRoot, true) });
+    const projects = [];
+    for (const projectId of new Set(input.tasks.map((t) => t.benchmarkProject))) {
+        const root = await resolveControlledBenchmarkProject(input.boundary.toolRoot, projectId);
+        const physicalProjectRoot = await establishPhysicalRoot(root);
+        const paths = await scanProjectTree(root, [".git"]);
+        const files = [];
+        for (const file of paths)
+            files.push({ path: file, bytes: await currentBytes(resolveWithinRoot(root, file), physicalProjectRoot, true) });
+        projects.push({ projectId, files });
+    }
+    const provider = input.provider;
+    const allowed = provider.kind === "deterministic-fixture" ? ["kind"] : provider.kind === "trusted-simulated-provider" ? ["kind", "simulationId", "providerId", "timeoutMs", "repairAttempts", "runAgent", "contextDependencies"] : ["kind", "providerId", "timeoutMs", "repairAttempts", "includeRealAgents", "trustedExecutable", "kitCommand", "contextDependencies", "env"];
+    campaignAssert(Object.keys(provider).every(key => allowed.includes(key)), "INCONSISTENT_EXECUTION_MODE");
+    const simulated = provider.kind === "trusted-simulated-provider";
+    campaignAssert(["deterministic-fixture", "trusted-simulated-provider", "real-agent"].includes(provider.kind), "INVALID_EXECUTION_MODE");
+    if (simulated)
+        campaignAssert(provider.runAgent !== runAgentPrompt && /^[A-Za-z0-9._-]+$/.test(provider.simulationId), "SIMULATION_CONTRACT_INVALID");
+    const real = provider.kind === "real-agent";
+    if (real)
+        campaignAssert(provider.includeRealAgents === undefined || provider.includeRealAgents === true, "REAL_PROVIDER_AUTHORIZATION_REQUIRED");
+    const observed = real && provider.includeRealAgents === true && provider.trustedExecutable ? await trustedProviderObservation(provider.trustedExecutable, provider.providerId) : null;
+    let retrievalVersion = null;
+    if (real && observed !== null) {
+        const commandsDir = await mkdtemp(path.join(os.tmpdir(), "lab-campaign-kit-proof-"));
+        try {
+            retrievalVersion = (await (provider.contextDependencies ?? defaultContextDependencies()).probeVersion({ kitCommand: provider.kitCommand ?? DEFAULT_AGENT_SUCCESS_KIT_COMMAND, commandsDir })).version ?? null;
+            campaignAssert(typeof retrievalVersion === "string" && retrievalVersion.length > 0 && retrievalVersion.length <= 256, "RETRIEVAL_IDENTITY_UNVERIFIABLE");
+        }
+        finally {
+            await rm(commandsDir, { recursive: true, force: true });
+        }
+    }
+    const functions = (object) => Object.entries(object).map(([key, value]) => ({ key, source: String(value) })).sort((a, b) => a.key < b.key ? -1 : 1);
+    return createCampaignIdentity({
+        tasks: input.tasks,
+        corpusBytes: input.corpusFile === null ? null : await currentBytes(input.corpusFile, physicalCorpusRoot), projects, sourceFiles,
+        config: provider.kind === "deterministic-fixture" ? { providerId: provider.kind, timeoutMs: 1000, repairAttempts: 0 } : { providerId: provider.providerId, timeoutMs: provider.timeoutMs, repairAttempts: provider.repairAttempts },
+        contextConfiguration: { contract: "existing-agent-success-context-v1", treatments: [...input.treatments] },
+        retrievalConfiguration: { contract: simulated ? "trusted-simulated-retrieval-v1" : real ? "real-provider-retrieval-v1" : "no-retrieval-fixture-v1", dependencies: simulated ? functions(provider.contextDependencies) : real ? functions(provider.contextDependencies ?? defaultContextDependencies()) : [], ...(real ? { kitCommand: provider.kitCommand ?? DEFAULT_AGENT_SUCCESS_KIT_COMMAND, version: retrievalVersion } : {}), executionDependencies: functions(input.dependencies ?? {}) },
+        verifiedProviderObservation: simulated ? Buffer.from(canonicalCampaignJson({ contract: "trusted-simulated-provider-v1", simulationId: provider.simulationId, implementation: String(provider.runAgent) })) : observed
+    });
+}
+/** Reads trusted current-byte inputs itself. The capability cannot be constructed from a claimed verified identity. */
+export async function prepareAgentSuccessCampaign(input) {
+    const snapshot = { ...input, boundary: { ...input.boundary }, tasks: structuredClone(input.tasks), treatments: [...input.treatments], provider: { ...input.provider }, dependencies: input.dependencies ? { ...input.dependencies } : undefined };
+    const identity = await produceIdentity(snapshot);
+    const prepared = Object.freeze({ identity: structuredClone(identity) });
+    preparations.set(prepared, snapshot);
+    return prepared;
+}
+/** Internal v2 gate: retained producer capability and newly observed current bytes are both required. */
+export async function validatePreparedCampaign(prepared) {
+    const input = preparations.get(prepared);
+    campaignAssert(input, "IDENTITY_PRODUCER_REQUIRED");
+    const identity = await produceIdentity(input);
+    campaignAssert(compareCampaignIdentity(identity, prepared.identity) === "MATCH", "IDENTITY_PROOF_MISSING_OR_CHANGED");
+    return { input: { ...input, boundary: { ...input.boundary }, tasks: structuredClone(input.tasks), treatments: [...input.treatments], provider: { ...input.provider }, dependencies: input.dependencies ? { ...input.dependencies } : undefined }, identity };
+}
+/** Explicit resumable internal API; the Prompt4 entrypoint and its default behavior stay intact. */
+export async function executeAgentSuccessCampaignV2(prepared, args) {
+    return executeResumableCampaign(prepared, args);
+}
+/** Internal entrypoint only. New campaign dispatch; reopening any existing generation belongs to Prompt 5. */
+export async function executeAgentSuccessCampaign(prepared, args) {
+    const outcome = { status: "failed", checkpoint: null, treatmentEvidence: [], completedCaseIds: [], errors: [], scientificArtifactsWritten: false };
+    const input = preparations.get(prepared);
+    if (!input) {
+        outcome.errors.push("IDENTITY_PRODUCER_REQUIRED");
+        return outcome;
+    }
+    let identity;
+    try {
+        identity = await produceIdentity(input);
+    }
+    catch {
+        outcome.errors.push("IDENTITY_INPUT_UNAVAILABLE");
+        return outcome;
+    }
+    if (compareCampaignIdentity(identity, prepared.identity) !== "MATCH") {
+        outcome.errors.push("IDENTITY_PROOF_MISSING_OR_CHANGED");
+        return outcome;
+    }
+    // Preserve historical proof diagnostics, then block live mode before any v1 state or dispatch.
+    if (input.provider.kind === "real-agent") {
+        outcome.errors.push("REAL_PROVIDER_REQUIRES_V2");
+        return outcome;
+    }
+    const { boundary, tasks, provider } = input;
+    const queue = buildCampaignQueue(tasks, input.treatments, args.limit);
+    let ownership;
+    let packBuilder;
+    let ownedRuntimeRoot;
+    const patchFiles = [], contextFiles = [], cases = [];
+    const at = () => new Date().toISOString();
+    const publish = async (checkpoint) => {
+        await writeCampaignCheckpoint(boundary, checkpoint, tasks, (phase) => args.hooks?.publication?.(checkpoint.generation, phase), ownership);
+        outcome.checkpoint = checkpoint;
+    };
+    try {
+        ownership = await acquireCampaignDispatch(boundary, identity);
+        const existing = await readCampaignCheckpoint(boundary, identity, tasks);
+        if (existing.checkpoint || existing.temporaryFiles.length) {
+            outcome.checkpoint = existing.checkpoint;
+            outcome.status = "resume-not-yet-supported";
+            return outcome;
+        }
+        await publish(createCampaignCheckpoint(identity, tasks, at()));
+        const runtimeRoot = await resolveSandboxRuntimeRoot(boundary.outputRoot, boundary.toolRoot);
+        try {
+            await lstat(runtimeRoot);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+            ownedRuntimeRoot = runtimeRoot;
+        }
+        if (provider.kind === "trusted-simulated-provider")
+            packBuilder = new AgentSuccessPackContextBuilder("trusted-simulated-kit", provider.contextDependencies);
+        for (let i = 0; i < queue.length; i += 2) {
+            const task = queue[i].task;
+            const forbiddenByTask = new Map();
+            const executionTask = provider.kind === "deterministic-fixture" ? task : sealTask(task, forbiddenByTask);
+            const canonicalProjectRoot = await resolveControlledBenchmarkProject(boundary.toolRoot, task.benchmarkProject);
+            const contextInput = buildContextTaskInput(executionTask);
+            const contexts = packBuilder ? { "raw-full-file": await buildRawFullFileContext(contextInput, canonicalProjectRoot), "context-pack": await packBuilder.build(contextInput, canonicalProjectRoot) } : null;
+            const treatments = [];
+            for (const unit of queue.slice(i, i + 2)) {
+                await assertCampaignDispatch(boundary, identity, ownership);
+                // Reverify current-byte identity before every dispatch, including the second treatment.
+                campaignAssert(compareCampaignIdentity(identity, await produceIdentity(input)) === "MATCH", "IDENTITY_CHANGED_BEFORE_DISPATCH");
+                campaignAssert(outcome.checkpoint.units[unit.checkpointIndex].state === "pending", "UNIT_NOT_PENDING");
+                await publish(transitionCampaignCheckpoint(outcome.checkpoint, tasks, unit.checkpointIndex, "in-progress", at()));
+                const common = { task: executionTask, treatmentId: unit.treatmentId, runId: args.runId, canonicalProjectRoot, runtimeRoot, privateRoots: [boundary.outputRoot, boundary.toolRoot], dependencies: input.dependencies };
+                const result = provider.kind === "trusted-simulated-provider" && contexts ? await executeAgentSuccessRepairTreatment({ ...common, repair: {
+                        providerId: provider.providerId, repairAttempts: provider.repairAttempts,
+                        createProposalSource: ({ attemptNumber, feedback }) => createRealAgentProposalSource({
+                            settings: { providerId: provider.providerId, timeoutMs: provider.timeoutMs, outDir: boundary.outputRoot, privateRoots: common.privateRoots, protectedRoots: [boundary.toolRoot, boundary.outputRoot, runtimeRoot, canonicalProjectRoot],
+                                runAgent: async (request) => {
+                                    await assertCampaignDispatch(boundary, identity, ownership);
+                                    await prepareCampaignProviderOutput(boundary, identity, ownership, agentSuccessAgentArtifactDirectory(task.benchmarkProject, task.id, unit.treatmentId, attemptNumber));
+                                    return provider.runAgent(request);
+                                } },
+                            runId: args.runId, task: projectAgentFacingTask(executionTask), contexts,
+                            forbidden: forbiddenByTask.get(task.id) ?? [], attemptNumber,
+                            ...(feedback ? { repair: { feedback, maxAttempts: 1 + provider.repairAttempts } } : {})
+                        })
+                    } }) : await executeAgentSuccessTreatment(common);
+                outcome.treatmentEvidence.push({ caseId: task.id, evidence: result.evidence });
+                treatments.push(result.evidence);
+                const limit = result.evidence.realAgent?.providerStatus === "agent-limit-reached";
+                if (limit || result.evidence.status !== "completed" || result.evidence.availability !== "complete") {
+                    await publish(transitionCampaignCheckpoint(outcome.checkpoint, tasks, unit.checkpointIndex, limit ? "paused-limit" : "incomplete", at(), null, limit ? "agent-limit-reached" : null));
+                    outcome.status = limit ? "provider-limit" : "incomplete";
+                    return outcome;
+                }
+                await persistCampaignSupportingArtifacts(boundary, identity, ownership, task, result);
+                const completion = await prepareCampaignCompletion(boundary, identity, task, result.evidence);
+                await publish(transitionCampaignCheckpoint(outcome.checkpoint, tasks, unit.checkpointIndex, "completed", at(), completion));
+                patchFiles.push(...result.patchFiles);
+                contextFiles.push(...result.contextFiles);
+            }
+            outcome.completedCaseIds.push(task.id);
+            cases.push({ caseId: task.id, caseName: task.title, benchmarkProject: task.benchmarkProject, taskLocality: task.taskLocality, fixtureId: provider.kind === "deterministic-fixture" ? task.deterministicFixture.id : "", treatments });
+        }
+        const complete = outcome.checkpoint.units.every((u) => u.state === "completed");
+        outcome.status = complete ? "completed" : "partial";
+        if (complete) {
+            const executionMode = provider.kind === "deterministic-fixture" ? "deterministic-fixture" : "real-agent";
+            const analysis = analyzeAgentSuccessRate(tasks, cases, executionMode);
+            const common = { runId: args.runId, pluginId: "agent-success-rate", pluginSchemaVersion: "1.0.0", startedAt: outcome.checkpoint.createdAt, completedAt: at() };
+            const execution = buildAgentSuccessRateExecutionArtifact({ ...common, cases, ...(provider.kind === "deterministic-fixture" ? {} : { realAgent: { providerId: provider.providerId, timeoutMs: provider.timeoutMs, repairAttempts: provider.repairAttempts, attemptsPerTreatment: (1 + provider.repairAttempts), promptTransport: "stdin", contextEffectEvaluated: analysis.contextEffectEvaluated } }) });
+            const scientific = buildAgentSuccessRateAnalysisArtifact({ ...common, analysis });
+            campaignAssert([...validateAgentSuccessRateExecutionArtifact(execution), ...validateAgentSuccessRateAnalysisArtifact(scientific), ...validateAgentSuccessRateArtifactFamily(execution, scientific)].length === 0, "SCIENTIFIC_ARTIFACT_INVALID");
+            const targets = [...contextFiles, ...patchFiles].map((f) => ({ filePath: resolveWithinRoot(boundary.outputRoot, f.relativePath), content: f.content }));
+            targets.push({ filePath: resolveWithinRoot(boundary.outputRoot, AGENT_SUCCESS_RATE_EXECUTION_ARTIFACT_FILE), content: `${JSON.stringify(execution, null, 2)}\n` }, { filePath: resolveWithinRoot(boundary.outputRoot, AGENT_SUCCESS_RATE_ANALYSIS_ARTIFACT_FILE), content: `${JSON.stringify(scientific, null, 2)}\n` });
+            await writeAgentSuccessRateArtifactTargets(targets, {
+                ensureDirectory: async () => { },
+                exists: async (file) => { try {
+                    await lstat(file);
+                    return true;
+                }
+                catch (error) {
+                    if (error.code === "ENOENT")
+                        return false;
+                    throw error;
+                } },
+                writeFile: (file, content) => persistCampaignArtifact(boundary, identity, ownership, path.relative(boundary.outputRoot, file).replace(/\\/g, "/"), content),
+                // Durable campaign evidence is never rolled back by final scientific publication.
+                removeFile: async () => { }
+            }, async (target) => {
+                await persistCampaignArtifact(boundary, identity, ownership, path.relative(boundary.outputRoot, target.filePath).replace(/\\/g, "/"), target.content, outcome.checkpoint);
+                return true;
+            });
+            outcome.scientificArtifactsWritten = true;
+        }
+    }
+    catch (error) {
+        outcome.status = "failed";
+        outcome.errors.push(boundMessage(error instanceof Error ? error.message : "CAMPAIGN_FAILED", [boundary.outputRoot, boundary.toolRoot, input.sourceRoot ?? ""]));
+        // Publication can fail after committing. Read the durable truth; never dispatch again in this invocation.
+        if (ownership)
+            try {
+                outcome.checkpoint = (await readCampaignCheckpoint(boundary, identity, tasks)).checkpoint;
+            }
+            catch {
+                outcome.errors.push("CHECKPOINT_READ_FAILED");
+            }
+    }
+    finally {
+        if (packBuilder)
+            try {
+                const reason = await packBuilder.dispose();
+                if (reason) {
+                    outcome.errors.push("CONTEXT_CLEANUP_FAILED");
+                    outcome.status = "failed";
+                }
+            }
+            catch {
+                outcome.errors.push("CONTEXT_CLEANUP_FAILED");
+                outcome.status = "failed";
+            }
+        if (ownedRuntimeRoot)
+            try {
+                await rmdir(ownedRuntimeRoot);
+            }
+            catch (error) {
+                if (error.code !== "ENOENT") {
+                    outcome.errors.push("RUNTIME_ROOT_NOT_REMOVED");
+                    outcome.status = "failed";
+                }
+            }
+        if (ownership)
+            try {
+                await releaseCampaignDispatch(boundary, identity, ownership);
+            }
+            catch {
+                outcome.errors.push("DISPATCH_RELEASE_FAILED");
+                outcome.status = "failed";
+            }
+    }
+    return outcome;
+}

package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import { validateAgentSuccessTask } from "../../../evaluation/agentSuccess/index.js";
+import { AGENT_SUCCESS_RATE_TREATMENT_IDS } from "./metadata.js";
+export const CAMPAIGN_IDENTITY_VERSION = "my-dev-kit-lab-agent-success-rate-campaign-identity-v1";
+export const CAMPAIGN_COMPONENTS = ["corpus", "tasks", "projects", "checks", "behavior", "context", "retrieval", "execution", "providerBuild"];
+export const CAMPAIGN_SOURCE_FILES = ["src/agents/types.ts", "src/agents/normalizeAgentTelemetry.ts", "src/agents/parseAgentTokenUsage.ts", "src/agents/runAgentPrompt.ts", "src/agents/adapters/codexAdapter.ts", "src/agents/adapters/claudeAdapter.ts", "src/evaluation/classifyAgentRunOutcome.ts", "src/experiments/plugins/agentSuccessRate/execution.ts", "src/experiments/plugins/agentSuccessRate/realAgentExecution.ts", "src/experiments/plugins/agentSuccessRate/contextGeneration.ts", "src/experiments/plugins/agentSuccessRate/repairExecution.ts", "src/experiments/plugins/agentSuccessRate/repairPolicy.ts", "src/experiments/plugins/agentSuccessRate/analysis.ts", "src/experiments/plugins/agentSuccessRate/plugin.ts"];
+/** Additional transitive execution/safety owners; a missing source contract cannot certify reuse. */
+export const CAMPAIGN_DEPENDENCY_FILES = ["src/experiments/plugins/agentSuccessRate/executionTypes.ts", "src/experiments/plugins/agentSuccessRate/analysisTypes.ts", "src/experiments/plugins/agentSuccessRate/attemptEvidence.ts", "src/experiments/plugins/agentSuccessRate/campaignIdentity.ts", "src/experiments/plugins/agentSuccessRate/campaignCheckpoint.ts", "src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.ts", "src/evaluation/agentSuccess/patchTypes.ts", "src/evaluation/agentSuccess/taskTypes.ts", "src/evaluation/agentSuccess/verificationTypes.ts", "src/evaluation/agentSuccess/corpusTypes.ts", "src/evaluation/benchmarkSandbox/types.ts", "src/evaluation/benchmarkSandbox/errors.ts", "src/evaluation/changeSet/types.ts", "src/agents/agentRegistry.ts", "src/core/pathSafety.ts", "src/core/runMeasuredCommand.ts", "src/core/resolveCommand.ts", "src/core/processTree.ts", "src/core/fileGlobs.ts", "src/core/countTokens.ts", "src/core/commandLine.ts", "src/runtime/managedProcess.ts", "src/evaluation/runMyDevKitRetrieval.ts", "src/evaluation/agentSuccess/assessBaseline.ts", "src/evaluation/agentSuccess/applyPatchToSandbox.ts", "src/evaluation/agentSuccess/extractPatchCandidate.ts", "src/evaluation/agentSuccess/parseUnifiedDiff.ts", "src/evaluation/agentSuccess/runVerificationCheck.ts", "src/evaluation/agentSuccess/taskPaths.ts", "src/evaluation/agentSuccess/validatePatchPolicy.ts", "src/evaluation/agentSuccess/validateAgentSuccessTask.ts", "src/evaluation/agentSuccess/readAgentSuccessCorpus.ts", "src/evaluation/agentSuccess/validateAgentSuccessCorpus.ts", "src/evaluation/benchmarkSandbox/createBenchmarkSandbox.ts", "src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.ts", "src/evaluation/benchmarkSandbox/pathPolicy.ts", "src/evaluation/benchmarkSandbox/treeSnapshot.ts", "src/evaluation/benchmarkSandbox/gitExecutor.ts", "src/evaluation/benchmarkSandbox/minimalHostEnv.ts", "src/evaluation/changeSet/captureChangeSet.ts", "src/experiments/plugins/agentSuccessRate/agentTaskProjection.ts", "src/experiments/plugins/agentSuccessRate/realAgentPrompt.ts", "src/experiments/plugins/agentSuccessRate/repairFeedback.ts", "src/experiments/plugins/agentSuccessRate/repairAnalysis.ts", "src/experiments/plugins/agentSuccessRate/metadata.ts", "src/experiments/plugins/agentSuccessRate/config.ts", "src/experiments/plugins/agentSuccessRate/executionArtifact.ts", "src/experiments/plugins/agentSuccessRate/analysisArtifact.ts", "src/experiments/outputPaths.ts"];
+export function campaignAssert(condition, code = "INVALID_CHECKPOINT") {
+    if (!condition)
+        throw new Error(code);
+}
+export function campaignHash(bytes) { return createHash("sha256").update(bytes).digest("hex"); }
+export function isCampaignDigest(value) { return typeof value === "string" && /^[a-f0-9]{64}$/.test(value); }
+export function campaignIdentifier(value) { return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value) && value !== "." && value !== ".."; }
+/** Bounded finite JSON canonicalization; arrays retain semantic order. No implicit undefined/NaN coercion. */
+export function canonicalCampaignJson(value) {
+    let visited = 0;
+    const encode = (v, depth) => {
+        campaignAssert(++visited <= 200_000 && depth <= 24, "DATA_BOUND");
+        if (v === null || typeof v === "boolean")
+            return JSON.stringify(v);
+        if (typeof v === "number") {
+            campaignAssert(Number.isFinite(v));
+            return JSON.stringify(v);
+        }
+        if (typeof v === "string") {
+            campaignAssert(v.length <= 1_048_576, "DATA_BOUND");
+            return JSON.stringify(v);
+        }
+        if (Array.isArray(v)) {
+            campaignAssert(v.length <= 10_000, "DATA_BOUND");
+            return `[${v.map((x) => encode(x, depth + 1)).join(",")}]`;
+        }
+        campaignAssert(typeof v === "object" && v !== null && Object.getPrototypeOf(v) === Object.prototype);
+        return `{${Object.keys(v).sort().map((k) => `${JSON.stringify(k)}:${encode(v[k], depth + 1)}`).join(",")}}`;
+    };
+    return encode(value, 0);
+}
+export function campaignExact(value, keys) {
+    campaignAssert(typeof value === "object" && value !== null && !Array.isArray(value));
+    campaignAssert(Object.keys(value).sort().join("\0") === [...keys].sort().join("\0"));
+}
+export function validateCampaignIdentity(value) {
+    campaignExact(value, ["schemaVersion", "algorithm", "pluginContract", "digest", "selected", "treatmentOrder", "config", "components"]);
+    campaignAssert(value.schemaVersion === CAMPAIGN_IDENTITY_VERSION && value.algorithm === "sha256" && value.pluginContract === "agent-success-rate/1.0.0", "INCOMPATIBLE_IDENTITY");
+    campaignAssert(Array.isArray(value.selected) && value.selected.length > 0 && value.selected.length <= 1000);
+    const ids = new Set();
+    for (const row of value.selected) {
+        campaignExact(row, ["caseId", "projectId", "taskDigest"]);
+        campaignAssert(campaignIdentifier(row.caseId) && campaignIdentifier(row.projectId) && isCampaignDigest(row.taskDigest) && !ids.has(row.caseId));
+        ids.add(row.caseId);
+    }
+    campaignAssert(canonicalCampaignJson(value.treatmentOrder) === canonicalCampaignJson(AGENT_SUCCESS_RATE_TREATMENT_IDS));
+    campaignExact(value.config, ["providerId", "timeoutMs", "repairAttempts"]);
+    campaignAssert(["codex", "claude", "deterministic-fixture"].includes(String(value.config.providerId)) && Number.isSafeInteger(value.config.timeoutMs) && Number(value.config.timeoutMs) > 0 && Number(value.config.timeoutMs) <= 1_800_000 && [0, 1, 2].includes(Number(value.config.repairAttempts)) && typeof value.config.repairAttempts === "number");
+    campaignExact(value.components, CAMPAIGN_COMPONENTS);
+    for (const name of CAMPAIGN_COMPONENTS) {
+        const component = value.components[name];
+        campaignExact(component, ["version", "digest"]);
+        campaignAssert(component.version === "1" && (component.digest === null || isCampaignDigest(component.digest)));
+    }
+    const { digest, ...body } = value;
+    campaignAssert(isCampaignDigest(digest) && digest === campaignHash(canonicalCampaignJson(body)), "IDENTITY_INTEGRITY");
+}
+/** Trusted caller supplies current corpus/project/source bytes. Never reads paths, executes a provider, or stores bodies. */
+export function createCampaignIdentity(input) {
+    campaignAssert(input.tasks.length > 0 && input.tasks.length <= 1000);
+    input.tasks.forEach((t) => campaignAssert(validateAgentSuccessTask(t).ok));
+    const digest = (v) => campaignHash(canonicalCampaignJson(v));
+    const manifest = (files) => {
+        campaignAssert(files.length <= 10_000 && new Set(files.map((f) => f.path)).size === files.length);
+        return files.map((f) => { campaignAssert(typeof f.path === "string" && f.path.length <= 512 && !f.path.includes("\\") && !f.path.includes(":") && !f.path.startsWith("/") && f.path.split("/").every((p) => p !== "" && p !== "." && p !== "..") && !f.path.includes("\0")); return { path: f.path, digest: campaignHash(f.bytes) }; }).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+    };
+    let source = null;
+    if (input.sourceFiles !== null) {
+        campaignAssert([...CAMPAIGN_SOURCE_FILES, ...CAMPAIGN_DEPENDENCY_FILES].every((required) => input.sourceFiles.some((f) => f.path === required)), "SOURCE_CONTRACT_MISSING");
+        source = digest(manifest(input.sourceFiles));
+    }
+    let projects = null;
+    if (input.projects !== null) {
+        campaignAssert([...input.projects.map((p) => p.projectId)].sort().join("\0") === [...new Set(input.tasks.map((t) => t.benchmarkProject))].sort().join("\0"));
+        projects = digest(input.projects.map((p) => ({ projectId: p.projectId, files: manifest(p.files) })).sort((a, b) => a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0));
+    }
+    const values = { corpus: input.corpusBytes === null ? null : campaignHash(input.corpusBytes), tasks: digest(input.tasks), projects, checks: digest(input.tasks.map((t) => ({ id: t.id, task: t.taskChecks, regression: t.regressionChecks, protected: t.protectedFiles }))), behavior: digest(input.tasks.map((t) => ({ id: t.id, facts: t.behaviorFacts, expected: t.expectedEditFiles, allowed: t.allowedEditFiles }))), context: input.contextConfiguration === null ? null : digest(input.contextConfiguration), retrieval: input.retrievalConfiguration === null ? null : digest(input.retrievalConfiguration), execution: source, providerBuild: input.config.providerId === "deterministic-fixture" ? digest("no-provider-invocation-v1") : input.verifiedProviderObservation === null ? null : campaignHash(input.verifiedProviderObservation) };
+    const body = { schemaVersion: CAMPAIGN_IDENTITY_VERSION, algorithm: "sha256", pluginContract: "agent-success-rate/1.0.0", selected: input.tasks.map((t) => ({ caseId: t.id, projectId: t.benchmarkProject, taskDigest: digest(t) })), treatmentOrder: [...AGENT_SUCCESS_RATE_TREATMENT_IDS], config: structuredClone(input.config), components: Object.fromEntries(CAMPAIGN_COMPONENTS.map((key) => [key, { version: "1", digest: values[key] }])) };
+    const result = { ...body, digest: digest(body) };
+    validateCampaignIdentity(result);
+    return result;
+}
+export function compareCampaignIdentity(left, right) {
+    try {
+        validateCampaignIdentity(left);
+        validateCampaignIdentity(right);
+    }
+    catch {
+        return "UNVERIFIABLE";
+    }
+    if (canonicalCampaignJson(left.selected) !== canonicalCampaignJson(right.selected) || canonicalCampaignJson(left.config) !== canonicalCampaignJson(right.config))
+        return "MISMATCH";
+    let missing = false;
+    for (const name of CAMPAIGN_COMPONENTS) {
+        const a = left.components[name];
+        const b = right.components[name];
+        if (a.digest === null || b.digest === null)
+            missing = true;
+        else if (a.version !== b.version || a.digest !== b.digest)
+            return "MISMATCH";
+    }
+    return missing ? "UNVERIFIABLE" : "MATCH";
+}

package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js

@@ -0,0 +1,175 @@
+import { lstat, readFile } from "node:fs/promises";
+import path from "node:path";
+import { isSameOrInside, resolvePhysicalPath } from "../../../evaluation/benchmarkSandbox/pathPolicy.js";
+import { normalizeAgentDurationV091 as normalizedDuration, normalizeAgentTokenUsageV091 as normalizedTokens } from "../../../agents/normalizeAgentTelemetry.js";
+import { validateAgentSuccessRateExecutionArtifact } from "./executionArtifact.js";
+import { validateAgentSuccessRateAnalysisArtifact, validateAgentSuccessRateArtifactFamily } from "./analysisArtifact.js";
+import { readCampaignCheckpointV2 } from "./campaignCheckpointStore.js";
+import { campaignAssert, campaignExact, campaignHash, campaignIdentifier, canonicalCampaignJson, isCampaignDigest } from "./campaignIdentity.js";
+import { CAMPAIGN_CHECKPOINT_SCHEMA_V2 } from "./campaignRecovery.js";
+import { campaignAttemptTelemetry, classifyCampaignEvidence, CAMPAIGN_QUANTITIES, QUALITY_REASONS } from "./campaignEvidenceQuality.js";
+export const CAMPAIGN_PROGRESS_SCHEMA = "my-dev-kit-lab-agent-success-rate-campaign-progress-v1";
+export const CAMPAIGN_UNIT_STATES = ["pending", "in-progress", "completed", "paused-limit", "incomplete", "uncertain"];
+export const CAMPAIGN_LIMITATIONS = [
+    "Completed evaluation does not mean successful task outcome. Trusted task checks determine success.",
+    "Matched subset values exclude unfinished or unavailable cases; pending outcomes are never failures.",
+    "Differences are context-pack minus raw-full-file, with no ranking, composite score, significance test or causal inference.",
+    "Duration is Lab wall-clock time, not provider-native execution time.",
+    "Token channels do not prove direct measurement of every field. Estimates are not billing usage.",
+    "Historical provider-limit epochs are operational history; only the authorized evaluated epoch contributes scientific evidence.",
+    "Deterministic-fixture mode validates the evaluation pipeline; it measures no coding agent or context selection effect."
+];
+const snapshots = new WeakMap();
+export function assertValidatedCampaignProgress(snapshot, boundary) {
+    const proof = snapshots.get(snapshot);
+    campaignAssert(proof && proof.boundary.outputRoot === boundary.outputRoot && proof.boundary.toolRoot === boundary.toolRoot && proof.canonical === canonicalCampaignJson(snapshot.model), "UNVALIDATED_PROGRESS_SNAPSHOT");
+    validateCampaignProgress(snapshot.model);
+}
+function caseState(states) {
+    if (states.every(s => s === "completed"))
+        return "completed";
+    if (states.some(s => ["in-progress", "incomplete", "uncertain"].includes(s)))
+        return "blockedUncertain";
+    if (states.includes("paused-limit"))
+        return "paused";
+    return states.includes("completed") ? "partiallyCompleted" : "pending";
+}
+/** Filesystem validation is mandatory; an absent, corrupt or untrusted checkpoint never becomes empty progress. */
+export async function readCampaignProgress(boundary, identity, tasks) {
+    const stored = await readCampaignCheckpointV2(boundary, identity, tasks);
+    campaignAssert(stored.checkpoint && stored.identityVerification === "MATCH", "PROGRESS_CHECKPOINT_UNAVAILABLE");
+    const cp = stored.checkpoint;
+    // Presence is a blocker, never evidence of a stale owner or permission to remove its marker.
+    const present = async (file) => {
+        try {
+            const s = await lstat(file);
+            campaignAssert(s.isFile() && !s.isSymbolicLink(), "PROGRESS_OWNERSHIP_UNSAFE");
+            return true;
+        }
+        catch (e) {
+            if (e.code === "ENOENT")
+                return false;
+            throw e;
+        }
+    };
+    const publicationUnresolved = stored.temporaryFiles.length > 0 || await present(path.join(boundary.outputRoot, "campaign-checkpoints-v2", identity.digest, "writer.json"));
+    const dispatchPresent = await present(path.join(boundary.outputRoot, "campaign-dispatch.json"));
+    const cases = tasks.map((task, i) => {
+        const units = cp.units.slice(i * 2, i * 2 + 2);
+        return { caseId: task.id, state: caseState(units.map(u => u.state)), treatments: units.map(u => ({ treatmentId: u.treatmentId, state: u.state, evaluatedTaskSuccess: u.completion?.taskSuccess ?? null, epochCount: u.epochs.length, historicalProviderLimitCount: u.epochs.slice(0, -1).filter(e => e.terminal?.reason === "provider-limit").length,
+                diagnostics: (u.state === "paused-limit" ? ["PROVIDER_LIMIT"] : ["in-progress", "uncertain"].includes(u.state) ? ["INVOCATION_UNCERTAIN"] : u.state === "incomplete" ? ["INCOMPLETE_EVALUATION"] : []), attempts: u.completion ? campaignAttemptTelemetry(u.completion.evidence) : [] })) };
+    });
+    const unitCounts = Object.fromEntries(CAMPAIGN_UNIT_STATES.map(s => [s, cp.units.filter(u => u.state === s).length]));
+    const caseCounts = Object.fromEntries(["completed", "partiallyCompleted", "pending", "paused", "blockedUncertain"].map(s => [s, cases.filter(c => c.state === s).length]));
+    const knownUncertainty = caseCounts.blockedUncertain > 0 || publicationUnresolved || dispatchPresent;
+    const diagnostics = [];
+    if (publicationUnresolved)
+        diagnostics.push("CHECKPOINT_PUBLICATION_UNRESOLVED");
+    if (dispatchPresent)
+        diagnostics.push("DISPATCH_OWNERSHIP_PRESENT");
+    if (unitCounts["in-progress"] || unitCounts.uncertain)
+        diagnostics.push("INVOCATION_UNCERTAIN");
+    if (unitCounts.incomplete)
+        diagnostics.push("INCOMPLETE_EVALUATION");
+    if (unitCounts["paused-limit"])
+        diagnostics.push("PROVIDER_LIMIT");
+    const state = knownUncertainty ? "blocked-uncertain" : unitCounts["paused-limit"] ? "paused-limit" : unitCounts.completed === cp.units.length ? "complete" : unitCounts.completed ? "partial" : "pending";
+    const model = { schemaVersion: CAMPAIGN_PROGRESS_SCHEMA, campaignIdentity: cp.identity.digest, checkpointSchemaVersion: cp.schemaVersion, checkpointGeneration: cp.generation, checkpointRevision: cp.integrity, executionMode: cp.identity.config.providerId === "deterministic-fixture" ? "deterministic-fixture" : "provider", selectedCaseCount: tasks.length, totalTreatmentUnits: cp.units.length, treatmentOrder: ["raw-full-file", "context-pack"], unitCounts, caseCounts, state, completedCaseIds: cases.filter(c => c.state === "completed").map(c => c.caseId), unfinishedCaseIds: cases.filter(c => c.state !== "completed").map(c => c.caseId), cases, knownProviderLimit: unitCounts["paused-limit"] > 0, knownUncertainty, diagnostics, scientificCompleteness: unitCounts.completed === cp.units.length ? "complete" : unitCounts.completed ? "partial" : "unavailable", finalScientificArtifactsAvailable: cp.finalization !== null, comparisons: classifyCampaignEvidence(cp, tasks), limitations: [...CAMPAIGN_LIMITATIONS] };
+    validateCampaignProgress(model);
+    const snapshot = { model };
+    snapshots.set(snapshot, { boundary: { ...boundary }, canonical: canonicalCampaignJson(model), checkpoint: cp });
+    return snapshot;
+}
+/** Load only the existing finalized scientific family, bound to the exact snapshot's content proofs. */
+export async function readCampaignProgressScientificFamily(snapshot, boundary) {
+    assertValidatedCampaignProgress(snapshot, boundary);
+    const cp = snapshots.get(snapshot).checkpoint;
+    if (!cp.finalization)
+        return null;
+    const load = async (name) => {
+        const ref = cp.finalization.artifacts.find(a => a.path === name);
+        const file = path.join(boundary.outputRoot, name), stat = await lstat(file);
+        const [physicalRoot, physicalFile] = await Promise.all([resolvePhysicalPath(boundary.outputRoot), resolvePhysicalPath(file)]);
+        campaignAssert(stat.isFile() && !stat.isSymbolicLink() && stat.size === ref.bytes && isSameOrInside(physicalRoot, physicalFile) && path.dirname(physicalFile) === physicalRoot, "SCIENTIFIC_ARTIFACT_UNSAFE");
+        const bytes = await readFile(file);
+        campaignAssert(bytes.length === ref.bytes && campaignHash(bytes) === ref.sha256, "SCIENTIFIC_ARTIFACT_CHANGED");
+        return JSON.parse(bytes.toString("utf8"));
+    };
+    const execution = await load("agent-success-rate-execution.json");
+    const analysis = await load("agent-success-rate-analysis.json");
+    campaignAssert([...validateAgentSuccessRateExecutionArtifact(execution), ...validateAgentSuccessRateAnalysisArtifact(analysis), ...validateAgentSuccessRateArtifactFamily(execution, analysis)].length === 0, "SCIENTIFIC_FAMILY_INVALID");
+    return { execution, analysis };
+}
+/** Closed durable schema, including reconciliation of identities, counts, quality and measurement availability. */
+export function validateCampaignProgress(value) {
+    campaignExact(value, ["schemaVersion", "campaignIdentity", "checkpointSchemaVersion", "checkpointGeneration", "checkpointRevision", "executionMode", "selectedCaseCount", "totalTreatmentUnits", "treatmentOrder", "unitCounts", "caseCounts", "state", "completedCaseIds", "unfinishedCaseIds", "cases", "knownProviderLimit", "knownUncertainty", "diagnostics", "scientificCompleteness", "finalScientificArtifactsAvailable", "comparisons", "limitations"]);
+    const m = value;
+    campaignAssert(m.schemaVersion === CAMPAIGN_PROGRESS_SCHEMA && m.checkpointSchemaVersion === CAMPAIGN_CHECKPOINT_SCHEMA_V2 && isCampaignDigest(m.campaignIdentity) && isCampaignDigest(m.checkpointRevision), "INVALID_PROGRESS_SCHEMA");
+    const count = (n, max = 2000) => campaignAssert(Number.isSafeInteger(n) && Number(n) >= 0 && Number(n) <= max, "INVALID_PROGRESS_COUNT");
+    count(m.checkpointGeneration, 1_000_000);
+    campaignAssert(m.checkpointGeneration > 0);
+    count(m.selectedCaseCount, 1000);
+    campaignAssert(m.selectedCaseCount > 0 && m.totalTreatmentUnits === 2 * m.selectedCaseCount && JSON.stringify(m.treatmentOrder) === '["raw-full-file","context-pack"]');
+    campaignAssert(["provider", "deterministic-fixture"].includes(m.executionMode) && Array.isArray(m.cases) && m.cases.length === m.selectedCaseCount);
+    campaignExact(m.unitCounts, CAMPAIGN_UNIT_STATES);
+    campaignExact(m.caseCounts, ["completed", "partiallyCompleted", "pending", "paused", "blockedUncertain"]);
+    const ids = [], states = [];
+    for (const c of m.cases) {
+        campaignExact(c, ["caseId", "state", "treatments"]);
+        campaignAssert(campaignIdentifier(c.caseId) && !ids.includes(c.caseId) && Array.isArray(c.treatments) && c.treatments.length === 2);
+        ids.push(c.caseId);
+        c.treatments.forEach((t, i) => {
+            campaignExact(t, ["treatmentId", "state", "evaluatedTaskSuccess", "epochCount", "historicalProviderLimitCount", "diagnostics", "attempts"]);
+            campaignAssert(t.treatmentId === m.treatmentOrder[i] && CAMPAIGN_UNIT_STATES.includes(t.state));
+            states.push(t.state);
+            count(t.epochCount, 2);
+            count(t.historicalProviderLimitCount, 1);
+            campaignAssert(t.historicalProviderLimitCount < Math.max(1, t.epochCount) && (t.state === "completed" ? typeof t.evaluatedTaskSuccess === "boolean" : t.evaluatedTaskSuccess === null));
+            const expected = t.state === "paused-limit" ? ["PROVIDER_LIMIT"] : ["in-progress", "uncertain"].includes(t.state) ? ["INVOCATION_UNCERTAIN"] : t.state === "incomplete" ? ["INCOMPLETE_EVALUATION"] : [];
+            campaignAssert(JSON.stringify(t.diagnostics) === JSON.stringify(expected) && Array.isArray(t.attempts) && t.attempts.length <= 3 && (t.state === "completed" ? t.attempts.length > 0 : t.attempts.length === 0));
+            t.attempts.forEach((a, j) => {
+                campaignExact(a, ["attemptNumber", "providerInvoked", "duration", "tokens", "tokenInterpretation"]);
+                campaignAssert(a.attemptNumber === j + 1 && typeof a.providerInvoked === "boolean");
+                campaignExact(a.duration, ["durationMs", "source", "reason"]);
+                campaignAssert(canonicalCampaignJson(a.duration) === canonicalCampaignJson(normalizedDuration(a.duration.durationMs)));
+                campaignExact(a.tokens, ["source", "reliability", "evidenceLevel", "classificationReason", "numericAvailability", "condition", "totalTokens", "inputTokens", "outputTokens", "cachedInputTokens", "reasoningTokens", "numericReasons"]);
+                campaignAssert(["provider-reported", "cli-json", "agent-reported", "otel", "estimated-from-text", "unavailable"].includes(a.tokens.source) && ["high", "medium", "low", "unavailable"].includes(a.tokens.reliability));
+                const n = normalizedTokens({ source: a.tokens.source, ...(a.tokens.totalTokens === null ? {} : { totalTokens: a.tokens.totalTokens }), ...(a.tokens.inputTokens === null ? {} : { inputTokens: a.tokens.inputTokens }), ...(a.tokens.outputTokens === null ? {} : { outputTokens: a.tokens.outputTokens }) }, a.tokens.reliability);
+                campaignAssert(canonicalCampaignJson(a.tokens) === canonicalCampaignJson(n));
+                campaignAssert(a.tokenInterpretation === (a.tokens.source === "estimated-from-text" ? "estimated" : a.tokens.totalTokens === null ? "unavailable" : "recorded-with-channel-provenance"));
+            });
+        });
+        campaignAssert(c.state === caseState(c.treatments.map(t => t.state)));
+    }
+    for (const s of CAMPAIGN_UNIT_STATES) {
+        count(m.unitCounts[s]);
+        campaignAssert(m.unitCounts[s] === states.filter(t => t === s).length);
+    }
+    for (const s of Object.keys(m.caseCounts)) {
+        count(m.caseCounts[s]);
+        campaignAssert(m.caseCounts[s] === m.cases.filter(c => c.state === s).length);
+    }
+    campaignAssert(JSON.stringify(m.completedCaseIds) === JSON.stringify(m.cases.filter(c => c.state === "completed").map(c => c.caseId)) && JSON.stringify(m.unfinishedCaseIds) === JSON.stringify(m.cases.filter(c => c.state !== "completed").map(c => c.caseId)));
+    campaignAssert(m.knownProviderLimit === (m.unitCounts["paused-limit"] > 0) && typeof m.knownUncertainty === "boolean" && (m.caseCounts.blockedUncertain === 0 || m.knownUncertainty));
+    campaignAssert(Array.isArray(m.diagnostics) && new Set(m.diagnostics).size === m.diagnostics.length && m.diagnostics.every(d => ["CHECKPOINT_PUBLICATION_UNRESOLVED", "DISPATCH_OWNERSHIP_PRESENT", "INVOCATION_UNCERTAIN", "INCOMPLETE_EVALUATION", "PROVIDER_LIMIT"].includes(d)));
+    campaignAssert(m.knownUncertainty === (m.caseCounts.blockedUncertain > 0 || m.diagnostics.includes("CHECKPOINT_PUBLICATION_UNRESOLVED") || m.diagnostics.includes("DISPATCH_OWNERSHIP_PRESENT")) && m.diagnostics.includes("INVOCATION_UNCERTAIN") === (m.unitCounts["in-progress"] + m.unitCounts.uncertain > 0) && m.diagnostics.includes("INCOMPLETE_EVALUATION") === (m.unitCounts.incomplete > 0) && m.diagnostics.includes("PROVIDER_LIMIT") === m.knownProviderLimit);
+    campaignAssert(m.state === (m.knownUncertainty ? "blocked-uncertain" : m.knownProviderLimit ? "paused-limit" : m.unitCounts.completed === m.totalTreatmentUnits ? "complete" : m.unitCounts.completed ? "partial" : "pending"));
+    campaignAssert(m.scientificCompleteness === (m.unitCounts.completed === m.totalTreatmentUnits ? "complete" : m.unitCounts.completed ? "partial" : "unavailable") && typeof m.finalScientificArtifactsAvailable === "boolean" && (!m.finalScientificArtifactsAvailable || m.scientificCompleteness === "complete"));
+    campaignAssert(Array.isArray(m.comparisons) && m.comparisons.length === CAMPAIGN_QUANTITIES.length);
+    m.comparisons.forEach((q, i) => {
+        campaignExact(q, ["quantity", "basis", "measurementBasis", "unit", "quality", "selectedCaseCount", "matchedCaseIds", "excludedCaseIds", "rawFullFileMean", "contextPackMean", "meanDifference", "reasons", "reasonExplanations"]);
+        campaignAssert(q.quantity === CAMPAIGN_QUANTITIES[i] && q.selectedCaseCount === ids.length && Array.isArray(q.matchedCaseIds) && new Set(q.matchedCaseIds).size === q.matchedCaseIds.length);
+        campaignAssert(JSON.stringify(q.matchedCaseIds) === JSON.stringify(ids.filter(id => q.matchedCaseIds.includes(id))) && q.matchedCaseIds.every(id => m.completedCaseIds.includes(id)) && JSON.stringify(q.excludedCaseIds) === JSON.stringify(ids.filter(id => !q.matchedCaseIds.includes(id))));
+        const token = q.quantity.endsWith("Tokens"), duration = q.quantity.endsWith("Duration");
+        campaignAssert(q.unit === (token ? "tokens" : duration ? "ms" : q.quantity === "regressionFailureCount" ? "count" : "ratio"));
+        campaignAssert(q.basis === (q.quantity.startsWith("first") || q.quantity === "initialTaskSuccess" ? "first-attempt" : q.quantity.startsWith("total") ? "total-across-attempts" : "final-attempt") && q.measurementBasis === (token ? "recorded-token-channel" : duration ? "lab-wall-clock" : q.quantity.endsWith("TaskSuccess") ? "trusted-task-evaluation" : "trusted-check-evaluation"));
+        campaignAssert(q.quality === (q.matchedCaseIds.length === 0 ? "unavailable" : q.excludedCaseIds.length || token ? "partial" : "strong"));
+        for (const n of [q.rawFullFileMean, q.contextPackMean, q.meanDifference])
+            campaignAssert(q.quality === "unavailable" ? n === null : typeof n === "number" && Number.isFinite(n));
+        campaignAssert(Array.isArray(q.reasons) && new Set(q.reasons).size === q.reasons.length && q.reasons.every(r => Object.hasOwn(QUALITY_REASONS, r)));
+        campaignAssert(JSON.stringify(q.reasonExplanations) === JSON.stringify(q.reasons.map(r => QUALITY_REASONS[r])));
+        campaignAssert(q.reasons.includes("NO_MATCHED_OBSERVATIONS") === (q.matchedCaseIds.length === 0) && q.reasons.includes("SELECTED_CASES_EXCLUDED") === (q.excludedCaseIds.length > 0) && (!token || q.quality === "unavailable" || q.reasons.includes("TOKEN_FIELD_PROVENANCE_UNPROVEN")));
+    });
+    campaignAssert(JSON.stringify(m.limitations) === JSON.stringify(CAMPAIGN_LIMITATIONS));
+    canonicalCampaignJson(m);
+}