@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/case-pipeline.test.js

@@ -0,0 +1,851 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect, Layer, Schema } from "effect";
+import { makeCaseCheckpointStoreV1 } from "../case-checkpoint-store.js";
+import { checkpointDigestV1, makePipelineCaseContextV1, PIPELINE_STAGES } from "../case-pipeline-protocol.js";
+import { EvalProjectAuthoringError } from "../errors.js";
+import { RepositoryHiddenOracleV1 } from "../repository-oracle-protocol.js";
+import { evaluateRepositoryOracleScopeCoverageV1 } from "../repository-oracle-scope-policy.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { readCasePipelineStatusV1, runCasePipelineV1 } from "../services/case-pipeline/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+import { runPipelineAdmitStageV1 } from "../services/pipeline-challenge/service.js";
+import { runPipelineSeedStageV1 } from "../services/pipeline-seed/service.js";
+import { buildRepositoryBehaviorMapV1 } from "../services/repository-model/service.js";
+const execute = promisify(execFile);
+const model = "openai/gpt-6-astra";
+const dimension = {
+    id: "repository-behavior-repair",
+    description: "Repair user-observable repository behavior while preserving compatible behavior.",
+    includes: ["historical executable defects", "public behavior preservation"],
+    excludes: ["style-only changes"]
+};
+async function repository(t) {
+    const root = await mkdtemp(join(tmpdir(), "case-pipeline-clamp-"));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const git = async (...args) => (await execute("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
+    await git("init", "-q");
+    await git("config", "user.email", "eval@example.test");
+    await git("config", "user.name", "Eval Fixture");
+    await mkdir(join(root, "src"), { recursive: true });
+    await writeFile(join(root, "package.json"), `${JSON.stringify({ name: "clamp-fixture", private: true, type: "module", exports: "./src/index.js", scripts: { test: "node --test" } }, null, 2)}\n`);
+    await writeFile(join(root, "src/clamp.js"), "export const clamp = (value) => Math.min(100, value);\n");
+    await writeFile(join(root, "src/index.js"), 'export { clamp } from "./clamp.js";\n');
+    await writeFile(join(root, "src/clamp.test.js"), [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { clamp } from "./index.js";',
+        'test("clamps high values", () => assert.equal(clamp(101), 100));'
+    ].join("\n"));
+    await writeFile(join(root, "src/control.test.js"), 'import assert from "node:assert/strict"; import test from "node:test"; test("control", () => assert.equal(2, 2));\n');
+    await git("add", ".");
+    await git("commit", "-qm", "add upper clamp");
+    const initialCommit = await git("rev-parse", "HEAD");
+    await writeFile(join(root, "src/clamp.js"), "export const clamp = (value) => Math.max(0, Math.min(100, value));\n");
+    await writeFile(join(root, "src/clamp.test.js"), [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { clamp } from "./index.js";',
+        'test("clamps low values", () => assert.equal(clamp(-1), 0));',
+        'test("clamps high values", () => assert.equal(clamp(101), 100));',
+        'test("preserves values", () => assert.equal(clamp(25.5), 25.5));'
+    ].join("\n"));
+    await git("add", ".");
+    await git("commit", "-qm", "fix clamp lower bound");
+    const referenceCommit = await git("rev-parse", "HEAD");
+    const caseDirectory = join(await mkdtemp(join(tmpdir(), "case-pipeline-state-")), "case");
+    t.after(() => rm(join(caseDirectory, ".."), { recursive: true, force: true }));
+    return { root, initialCommit, referenceCommit, caseDirectory };
+}
+async function repositoryWithSupersededIncompatibleTest(t) {
+    const root = await mkdtemp(join(tmpdir(), "case-pipeline-current-surface-"));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const git = async (...args) => (await execute("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
+    await git("init", "-q");
+    await git("config", "user.email", "eval@example.test");
+    await git("config", "user.name", "Eval Fixture");
+    await mkdir(join(root, "packages/main/src"), { recursive: true });
+    await mkdir(join(root, "packages/extra/src"), { recursive: true });
+    await writeFile(join(root, "package.json"), `${JSON.stringify({ name: "current-surface-fixture", private: true, workspaces: ["packages/*"] }, null, 2)}\n`);
+    for (const name of ["main", "extra"]) {
+        await writeFile(join(root, `packages/${name}/package.json`), `${JSON.stringify({ name: `fixture-${name}`, private: true, type: "module", scripts: { test: "node --test" } }, null, 2)}\n`);
+    }
+    await writeFile(join(root, "packages/main/src/clamp.js"), "export const clamp = (value) => Math.min(100, value);\n");
+    await writeFile(join(root, "packages/main/src/clamp.test.js"), [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { clamp } from "./clamp.js";',
+        'test("clamps high values", () => assert.equal(clamp(101), 100));'
+    ].join("\n"));
+    await writeFile(join(root, "packages/extra/src/extra.test.js"), 'import assert from "node:assert/strict"; import test from "node:test"; test("extra control", () => assert.equal(2, 2));\n');
+    await git("add", ".");
+    await git("commit", "-qm", "add fixture packages");
+    const initialCommit = await git("rev-parse", "HEAD");
+    await writeFile(join(root, "packages/main/src/clamp.js"), [
+        "export const clamp = (value) => {",
+        '  if (typeof value !== "number") throw new TypeError("value must be a number");',
+        "  return Math.max(0, Math.min(100, value));",
+        "};",
+        ""
+    ].join("\n"));
+    await writeFile(join(root, "packages/main/src/clamp.test.js"), [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { clamp } from "./clamp.js";',
+        'test("rejects non-numbers", () => assert.throws(() => clamp("25"), TypeError));',
+        'test("clamps high values", () => assert.equal(clamp(101), 100));'
+    ].join("\n"));
+    await writeFile(join(root, "packages/extra/src/reference-only-helper.js"), "export const answer = 42;\n");
+    await writeFile(join(root, "packages/extra/src/extra.test.js"), [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { answer } from "./reference-only-helper.js";',
+        'test("reference-only helper", () => assert.equal(answer, 42));'
+    ].join("\n"));
+    await git("add", ".");
+    await git("commit", "-qm", "reject non-number clamp input");
+    const referenceCommit = await git("rev-parse", "HEAD");
+    await rm(join(root, "packages/extra"), { recursive: true });
+    await git("add", "-A");
+    await git("commit", "-qm", "remove superseded reference-only coverage");
+    const tipCommit = await git("rev-parse", "HEAD");
+    const caseDirectory = join(await mkdtemp(join(tmpdir(), "case-pipeline-current-surface-state-")), "case");
+    t.after(() => rm(join(caseDirectory, ".."), { recursive: true, force: true }));
+    return { root, initialCommit, referenceCommit, tipCommit, caseDirectory };
+}
+const fixture = (id, assertion, kind, expectationMode) => ({
+    id,
+    kind,
+    description: id,
+    expectedBehavior: id,
+    expectationMode,
+    source: "generated",
+    testPath: "src/clamp.test.js"
+});
+const overlay = (id, assertion) => ({
+    path: "src/clamp.test.js",
+    fixtureIds: [id],
+    content: [
+        'import assert from "node:assert/strict";',
+        'import test from "node:test";',
+        'import { clamp } from "./index.js";',
+        `test(${JSON.stringify(id)}, () => { ${assertion} });`
+    ].join("\n")
+});
+const suite = (caseId, overconstrainValidAlternative = false) => ({
+    version: 1,
+    caseId,
+    fixtures: [
+        fixture("negative", "", "historical-regression", "changes"),
+        fixture("upper", "", "boundary", "preserved"),
+        fixture("fraction", "", "metamorphic", "preserved"),
+        ...(overconstrainValidAlternative
+            ? [fixture("implementation-shape", "", "boundary", "preserved")]
+            : [])
+    ],
+    overlays: [
+        overlay("negative", "assert.equal(clamp(-1), 0)"),
+        overlay("upper", "assert.equal(clamp(101), 100)"),
+        overlay("fraction", "assert.equal(clamp(25.5), 25.5)"),
+        ...(overconstrainValidAlternative
+            ? [overlay("implementation-shape", 'assert.equal(Object.hasOwn(clamp, "prototype"), false)')]
+            : [])
+    ]
+});
+const correctA = "export const clamp = (value) => Math.min(100, Math.max(0, value));\n";
+const correctB = "export function clamp(value) { if (value < 0) return 0; if (value > 100) return 100; return value; }\n";
+const wrong = "export const clamp = (value) => Math.min(100, Math.max(0, Math.round(value)));\n";
+const heldOut = "export const clamp = (value) => Math.max(0, Math.min(100, Math.trunc(value)));\n";
+const edits = (source) => [{ path: "src/clamp.js", content: source }];
+test("automatic seed discovery retains comparative model ranking", async (t) => {
+    const repo = await repository(t);
+    const context = makePipelineCaseContextV1({
+        repositoryRoot: repo.root,
+        caseDirectory: repo.caseDirectory,
+        caseId: "pipeline-clamp-discovery",
+        dimension,
+        primaryModel: model,
+        budget: { maxModelCalls: 10, maxWallMs: 3_600_000 },
+        requestedRef: repo.referenceCommit,
+        oracleRepetitions: 2,
+        oracleConcurrency: 1
+    });
+    const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
+        repositoryRoot: context.repositoryRoot,
+        requestedRef: context.requestedRef ?? "HEAD"
+    }));
+    const scripted = transport();
+    const checkpoint = await Effect.runPromise(runPipelineSeedStageV1({
+        context,
+        map,
+        checkpoints: {},
+        checkpoint: () => Effect.void
+    }).pipe(Effect.provide(Layer.mergeAll(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(scripted.layer)), RepositoryFoundryProgressLive()))));
+    assert.equal(checkpoint.selection.selectedCommit, repo.referenceCommit);
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "PipelineSeedRankingV1").length, 1, "automatic discovery still ranks competing candidates with the model");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "PipelineSeedDimensionFitV1").length, 1, "the replay-qualified automatic candidate still receives complete dimension assessment");
+});
+test("explicit seed qualification uses the inspected tip's executable test surface", async (t) => {
+    const repo = await repositoryWithSupersededIncompatibleTest(t);
+    const context = makePipelineCaseContextV1({
+        repositoryRoot: repo.root,
+        caseDirectory: repo.caseDirectory,
+        caseId: "pipeline-clamp-current-surface",
+        dimension,
+        primaryModel: model,
+        budget: { maxModelCalls: 10, maxWallMs: 3_600_000 },
+        referenceCommit: repo.referenceCommit,
+        requestedRef: repo.tipCommit,
+        oracleRepetitions: 2,
+        oracleConcurrency: 1
+    });
+    const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
+        repositoryRoot: context.repositoryRoot,
+        requestedRef: context.requestedRef ?? "HEAD"
+    }));
+    const scripted = transport();
+    const checkpoint = await Effect.runPromise(runPipelineSeedStageV1({
+        context,
+        map,
+        checkpoints: {},
+        checkpoint: () => Effect.void
+    }).pipe(Effect.provide(Layer.mergeAll(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(scripted.layer)), RepositoryFoundryProgressLive()))));
+    assert.equal(checkpoint.selection.selectedCommit, repo.referenceCommit);
+    assert.deepEqual(checkpoint.seed.candidateTestIds, ["test-packages-main-src-clamp-test-js"]);
+    assert.equal(checkpoint.qualification.stability.stable, true);
+    assert.equal(checkpoint.qualification.seed.status, "qualified");
+});
+function reply(input, options) {
+    const parsed = JSON.parse(input.input);
+    switch (input.schemaName) {
+        case "PipelineSeedRankingV1": {
+            const candidates = parsed.candidates;
+            return {
+                candidates: candidates.map(({ commit }, index) => ({
+                    commit,
+                    score: 100 - index,
+                    rationale: "Executable historical clamp repair fits the dimension."
+                }))
+            };
+        }
+        case "PipelineSeedDimensionFitV1": {
+            const candidate = parsed.candidate;
+            const requested = parsed.dimension;
+            return {
+                commit: candidate.commit,
+                requestedDimension: requested.id,
+                outcome: "fit",
+                detail: "The replayed clamp repair directly exercises one coherent slice of the requested dimension and respects its exclusions.",
+                includes: requested.includes.map((requirement, index) => ({
+                    requirement,
+                    outcome: index === 0 ? "supports" : "does-not-support",
+                    detail: index === 0
+                        ? "The replayed behavior supplies this capability."
+                        : "This narrow historical fix does not need to cover every example in the dimension."
+                })),
+                excludes: requested.excludes.map((requirement) => ({
+                    requirement,
+                    outcome: "respected",
+                    detail: "The replayed behavior stays outside this exclusion."
+                }))
+            };
+        }
+        case "routekit_pipeline_spec_analysis_v1": {
+            const task = parsed.task;
+            const behaviorId = task.seed.targetBehavior[0].id;
+            return {
+                action: "submit",
+                result: {
+                    summary: "Clamp needs a lower bound while retaining its upper bound and exact in-range values.",
+                    userObservableBehaviors: [
+                        "negative values become zero",
+                        "upper values remain capped",
+                        "in-range values remain unchanged"
+                    ],
+                    constraints: ["preserve the clamp export"],
+                    risks: ["rounding would corrupt in-range values"],
+                    relevantPaths: ["src/clamp.js", "src/index.js"],
+                    scope: [
+                        {
+                            id: "lower-bound",
+                            kind: "changed",
+                            behaviorIds: [behaviorId],
+                            description: "Negative inputs return zero.",
+                            evidence: [
+                                { commit: "initial", path: "src/clamp.js", startLine: 1, endLine: 1 },
+                                { commit: "reference", path: "src/clamp.js", startLine: 1, endLine: 1 }
+                            ]
+                        },
+                        {
+                            id: "preserve-range",
+                            kind: "preserved",
+                            behaviorIds: [behaviorId],
+                            description: "In-range behavior remains unchanged.",
+                            evidence: [
+                                { commit: "initial", path: "src/clamp.js", startLine: 1, endLine: 1 },
+                                { commit: "reference", path: "src/clamp.js", startLine: 1, endLine: 1 }
+                            ]
+                        },
+                        {
+                            id: "preserve-upper",
+                            kind: "preserved",
+                            behaviorIds: [behaviorId],
+                            description: "Upper-bound behavior remains unchanged.",
+                            evidence: [
+                                { commit: "initial", path: "src/clamp.js", startLine: 1, endLine: 1 },
+                                { commit: "reference", path: "src/clamp.js", startLine: 1, endLine: 1 }
+                            ]
+                        }
+                    ]
+                }
+            };
+        }
+        case "routekit_pipeline_visible_task_v1":
+            return {
+                requestText: "Repair clamp so negative inputs return zero while values above one hundred remain capped and in-range values remain unchanged.",
+                constraints: ["Preserve the public clamp export."]
+            };
+        case "routekit_pipeline_specification_review_v1": {
+            const envelope = parsed;
+            const task = (envelope.task ?? parsed);
+            const source = task.sourceCatalog.find(({ path, sourceId }) => path === "src/clamp.js" && sourceId.length > 0);
+            const referenceSource = task.sourceCatalog.find(({ path, sourceId }) => path === "src/clamp.js" && sourceId !== source.sourceId) ?? source;
+            return {
+                action: "submit",
+                result: {
+                    verdict: "sufficient",
+                    detail: "The task covers the executed historical behavior and preservation boundary.",
+                    criticalBehaviorCoverage: task.targetBehavior
+                        .filter(({ critical }) => critical)
+                        .map(({ id }) => ({
+                        behaviorId: id,
+                        outcome: "covered",
+                        detail: "Explicitly stated."
+                    })),
+                    dimensionFit: {
+                        requestedDimension: dimension.id,
+                        outcome: "fit",
+                        detail: "Behavioral repository repair.",
+                        evidencePaths: ["src/clamp.js"]
+                    },
+                    referenceCompatibility: {
+                        outcome: "compatible",
+                        detail: "Reference implements all scope.",
+                        scopeCoverage: task.scope.map(({ id }) => ({
+                            scopeId: id,
+                            outcome: "supported",
+                            detail: "Supported by reference.",
+                            evidence: [{ sourceId: referenceSource.sourceId, startLine: 1, endLine: 1 }]
+                        }))
+                    },
+                    findings: []
+                }
+            };
+        }
+        case "routekit_pipeline_oracle_submission_v1":
+            return {
+                action: "submit",
+                result: {
+                    suite: (() => {
+                        const proposed = suite(parsed.task.visibleTask.id, options.oracleRejectsValidAlternative);
+                        return options.oracleMissesChangedScope ? {
+                            ...proposed,
+                            overlays: proposed.overlays.map((entry) => entry.fixtureIds.includes("negative")
+                                ? overlay("negative", "assert.equal(clamp(25), 25)") : entry)
+                        } : proposed;
+                    })(),
+                    scopeCoverage: [
+                        {
+                            scopeId: "lower-bound",
+                            outcome: "covered",
+                            fixtureIds: ["negative"],
+                            detail: "Negative assertion."
+                        },
+                        {
+                            scopeId: "preserve-range",
+                            outcome: "covered",
+                            fixtureIds: ["fraction"],
+                            detail: "In-range preservation assertion."
+                        },
+                        {
+                            scopeId: "preserve-upper",
+                            outcome: "covered",
+                            fixtureIds: ["upper"],
+                            detail: "Upper-bound preservation assertion."
+                        }
+                    ]
+                }
+            };
+        case "routekit_pipeline_oracle_coverage_review_v1":
+            return {
+                verdict: "approve",
+                detail: "Every scope has an executable assertion.",
+                scopeCoverage: [
+                    {
+                        scopeId: "lower-bound",
+                        outcome: "covered",
+                        detail: "negative assertion",
+                        counterexample: "",
+                        assertionEvidence: [{ fixtureId: "negative", excerpt: "assert.equal(clamp(-1), 0)" }]
+                    },
+                    {
+                        scopeId: "preserve-range",
+                        outcome: "covered",
+                        detail: "fraction assertion",
+                        counterexample: "",
+                        assertionEvidence: [
+                            { fixtureId: "fraction", excerpt: "assert.equal(clamp(25.5), 25.5)" }
+                        ]
+                    },
+                    {
+                        scopeId: "preserve-upper",
+                        outcome: "covered",
+                        detail: "upper assertion",
+                        counterexample: "",
+                        assertionEvidence: [{ fixtureId: "upper", excerpt: "assert.equal(clamp(101), 100)" }]
+                    }
+                ]
+            };
+        case "routekit_pipeline_oracle_coverage_witness_v1": {
+            const hypothesis = parsed.pendingCoverageHypothesis;
+            const lowerBound = hypothesis.scopeId === "lower-bound";
+            return {
+                witnesses: [
+                    {
+                        scopeId: hypothesis.scopeId,
+                        explanation: lowerBound
+                            ? "Removing the lower clamp violates only the changed lower-bound behavior."
+                            : "Rounding an in-range fraction violates only the preserved exact-value behavior.",
+                        fileOverrides: [
+                            {
+                                path: "src/clamp.js",
+                                edits: [
+                                    {
+                                        search: "export const clamp = (value) => Math.max(0, Math.min(100, value));",
+                                        replace: lowerBound
+                                            ? "export const clamp = (value) => Math.min(100, value);"
+                                            : "export const clamp = (value) => Math.max(0, Math.min(100, Math.round(value)));"
+                                    }
+                                ]
+                            }
+                        ],
+                        testPath: "src/clamp.test.js",
+                        testSource: overlay(`witness-${hypothesis.scopeId}`, lowerBound ? "assert.equal(clamp(-1), 0)" : "assert.equal(clamp(25.5), 25.5)").content,
+                        expectedBehavior: lowerBound
+                            ? "Negative values clamp to zero."
+                            : "In-range fractional values remain unchanged.",
+                        expectationMode: lowerBound ? "changes" : "preserved"
+                    }
+                ]
+            };
+        }
+        case "routekit_pipeline_valid_solution_generator_a_v1":
+            return options.oracleRejectsValidAlternative
+                ? {
+                    id: "solver-b",
+                    family: "guards",
+                    kind: "behavior-preserving-valid",
+                    fileOverrides: edits(correctB)
+                }
+                : {
+                    id: "solver-a",
+                    family: "composition",
+                    kind: "independent-valid",
+                    fileOverrides: edits(correctA)
+                };
+        case "routekit_pipeline_valid_solution_generator_b_v1":
+            return {
+                id: "solver-b",
+                family: "guards",
+                kind: "behavior-preserving-valid",
+                fileOverrides: edits(correctB)
+            };
+        case "routekit_pipeline_adversary_v1":
+            return {
+                id: "near-miss",
+                family: "partial-fix",
+                kind: "mutation",
+                fileOverrides: edits(wrong)
+            };
+        case "routekit_pipeline_control_correctness_review_v1": {
+            const task = parsed;
+            assert.ok(task.solutions.every(({ expectedClass }) => expectedClass === undefined), "fixture-blind critics do not receive proposal labels");
+            return {
+                critiques: task.solutions.map(({ id }) => ({
+                    solutionId: id,
+                    classification: id.startsWith("solver-") ? "valid" : "wrong",
+                    detail: id.startsWith("solver-")
+                        ? "The implementation satisfies the visible lower bound and preservation requirements."
+                        : "The implementation rounds an in-range value and violates the visible preservation requirement.",
+                    violatedBehaviorIds: id.startsWith("solver-") ? [] : [task.targetBehavior[0].id]
+                }))
+            };
+        }
+        case "routekit_pipeline_tournament_repair_v1": {
+            const task = parsed.task;
+            assert.deepEqual(task.measuredAdequacy.falseRejectedSolutionIds, options.oracleMissesChangedScope ? [] : ["solver-b"]);
+            if (options.oracleMissesChangedScope)
+                assert.deepEqual(task.measuredScopeGaps.unresolvedScopeIds, ["lower-bound"]);
+            const replacementSuite = suite(task.visibleTask.id);
+            const mixedPurposeSuite = options.mixedPurposeRepairFixture
+                ? {
+                    ...replacementSuite,
+                    fixtures: replacementSuite.fixtures.map((candidate) => candidate.id === "upper"
+                        ? { ...candidate, expectationMode: "changes" }
+                        : candidate)
+                }
+                : replacementSuite;
+            return {
+                action: "submit",
+                result: {
+                    targetedSolutionIds: options.oracleMissesChangedScope
+                        ? ["historical-reference", "historical-defect"] : ["solver-b"],
+                    suite: mixedPurposeSuite,
+                    rationale: "Remove the implementation-shape assertion while retaining all user-observable behavior assertions."
+                }
+            };
+        }
+        case "routekit_pipeline_held_out_adversary_v1":
+            return {
+                solutions: [
+                    {
+                        id: "held-out-rounding",
+                        family: "over-broad",
+                        kind: "model-near-miss",
+                        fileOverrides: edits(heldOut)
+                    }
+                ]
+            };
+        default:
+            throw new Error(`unhandled scripted request ${input.schemaName} (${input.operationId})`);
+    }
+}
+function transport(options = {}) {
+    const calls = [];
+    let interruptionStep = 0;
+    const completeDetailed = (input) => Effect.gen(function* () {
+        calls.push(input);
+        if (options.interruptOnceAt !== undefined &&
+            input.operationId.includes(options.interruptOnceAt)) {
+            interruptionStep += 1;
+            if (interruptionStep === 1) {
+                return yield* new EvalProjectAuthoringError({
+                    operation: "authoring-evaluations",
+                    detail: "Transport error: injected transport failure"
+                });
+            }
+            if (interruptionStep === 2) {
+                return yield* Effect.die("injected transport interruption after retry");
+            }
+        }
+        const response = reply(input, options);
+        const schemaProperties = input.jsonSchema.properties;
+        return {
+            text: JSON.stringify(schemaProperties !== undefined && Object.hasOwn(schemaProperties, "turn")
+                ? { turn: response }
+                : response),
+            callId: `fake-${String(calls.length)}`
+        };
+    });
+    return {
+        calls,
+        layer: Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+            complete: (input) => Effect.map(completeDetailed(input), ({ text }) => text),
+            completeDetailed
+        }))
+    };
+}
+const runPipeline = async (input) => {
+    const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
+        repositoryRoot: input.context.repositoryRoot,
+        requestedRef: input.context.requestedRef ?? "HEAD"
+    }));
+    return Effect.runPromise(runCasePipelineV1({
+        context: input.context, map,
+        ...(input.store === undefined ? {} : { store: input.store })
+    }).pipe(Effect.provide(Layer.mergeAll(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(input.transportLayer)), RepositoryFoundryProgressLive()))));
+};
+const contextFor = (repo, caseId, maxModelCalls) => makePipelineCaseContextV1({
+    repositoryRoot: repo.root,
+    caseDirectory: repo.caseDirectory,
+    caseId,
+    dimension,
+    primaryModel: model,
+    budget: { maxModelCalls, maxWallMs: 3_600_000 },
+    referenceCommit: repo.referenceCommit,
+    requestedRef: repo.referenceCommit,
+    oracleRepetitions: 2,
+    oracleConcurrency: 1
+});
+const receivedCalls = async (caseDirectory) => {
+    const store = makeCaseCheckpointStoreV1({ caseDirectory });
+    const ledger = await Effect.runPromise(store.readLedger);
+    return {
+        ledger,
+        received: ledger.filter(({ phase }) => phase === "received")
+    };
+};
+test("real pipeline admits from executed evidence and a rerun makes zero model calls", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp", 80);
+    const scripted = transport();
+    const first = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(first.state.status, "admitted", first.state.reason);
+    assert.equal(first.state.stage, "done");
+    assert.equal(first.admit?.verdict, "admitted");
+    assert.equal(first.admit?.benchmarkCase.status, "valid-library");
+    assert.ok((first.checkpoints.tournament?.oracle.observations.length ?? 0) > 0);
+    assert.ok((first.checkpoints.challenge?.oracle.observations.length ?? 0) > 0);
+    assert.deepEqual(first.checkpoints.oracle?.witnesses, []);
+    assert.equal(first.checkpoints.oracle?.scopeCoverage.find(({ scopeId }) => scopeId === "preserve-upper")
+        ?.outcome, "uncertain", "the scheduling ceiling advances to executed tournament evidence without self-attesting remaining scope coverage");
+    assert.ok((first.admit?.evidence.validCount ?? 0) >= 2);
+    assert.ok((first.admit?.evidence.wrongCount ?? 0) >= 1);
+    assert.equal(first.admit?.evidence.heldOutWrongCount, 1);
+    assert.ok((first.admit?.evidence.semanticKills.length ?? 0) >= 2);
+    assert.deepEqual(first.checkpoints.tournament?.valid.map(({ id }) => id), ["solver-a"], "one generated valid control supplements the executed historical reference");
+    assert.ok(first.checkpoints.tournament?.oracle.solutions.some(({ id, expectedClass }) => id === "historical-reference" && expectedClass === "valid"), "the development tournament retains the executed historical reference as independent valid evidence");
+    const challengeSolutionIds = first.checkpoints.challenge?.oracle.solutions.map(({ id }) => id) ?? [];
+    assert.ok(challengeSolutionIds.includes("historical-reference"));
+    assert.ok(challengeSolutionIds.includes("held-out-rounding"));
+    assert.ok(!challengeSolutionIds.includes("solver-a"), "the fresh challenge excludes development valid controls");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_oracle_coverage_review_v1").length, 0, "oracle authoring does not spend proactive critic calls before measured execution");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_oracle_coverage_witness_v1").length, 0, "oracle authoring does not spend proactive witness calls before measured execution");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_valid_solution_generator_a_v1").length, 1);
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_valid_solution_generator_b_v1").length, 0);
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_control_correctness_review_v1").length, 0, "fixture-blind control critics stay lazy when execution accepts the generated valid control");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_tournament_repair_v1").length, 0);
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "PipelineSeedRankingV1").length, 0, "an explicit reference has no candidate-ranking model call");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "PipelineSeedDimensionFitV1").length, 1, "the explicit reference still receives a complete model dimension assessment");
+    const callsBeforeScopeCheck = scripted.calls.length;
+    const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
+        repositoryRoot: context.repositoryRoot,
+        requestedRef: context.requestedRef ?? "HEAD"
+    }));
+    const invalidScopeAdmission = await Effect.runPromise(runPipelineAdmitStageV1({
+        context,
+        map,
+        checkpoints: {
+            ...first.checkpoints,
+            oracle: {
+                ...first.checkpoints.oracle,
+                scopeCoverage: first.checkpoints.oracle.scopeCoverage.map((coverage) => coverage.scopeId === "lower-bound" ? { ...coverage, fixtureIds: ["upper"] } : coverage)
+            }
+        },
+        checkpoint: () => Effect.void
+    }).pipe(Effect.provide(Layer.mergeAll(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(scripted.layer)), RepositoryFoundryProgressLive()))));
+    assert.equal(invalidScopeAdmission.verdict, "rejected");
+    assert.ok(invalidScopeAdmission.reasons.some((reason) => reason.includes("changed scope lower-bound lacks one mapped changes fixture with repeated historical-reference passes and attributed historical-defect failures")), "a model-authored mapping cannot substitute unrelated executed fixture evidence");
+    assert.equal(scripted.calls.length, callsBeforeScopeCheck, "executed scope verification adds no model calls");
+    const mixedPurposeFixtureAdmission = await Effect.runPromise(runPipelineAdmitStageV1({
+        context,
+        map,
+        checkpoints: {
+            ...first.checkpoints,
+            tournament: {
+                ...first.checkpoints.tournament,
+                oracle: {
+                    ...first.checkpoints.tournament.oracle,
+                    fixtures: first.checkpoints.tournament.oracle.fixtures.map((candidate) => candidate.id === "upper" && candidate.kind === "boundary"
+                        ? { ...candidate, expectationMode: "changes" }
+                        : candidate)
+                }
+            }
+        },
+        checkpoint: () => Effect.void
+    }).pipe(Effect.provide(Layer.mergeAll(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(scripted.layer)), RepositoryFoundryProgressLive()))));
+    assert.equal(mixedPurposeFixtureAdmission.verdict, "admitted", "a repeatedly passing fixture may prove a mapped preserved scope while its primary expectation also changes");
+    assert.equal(scripted.calls.length, callsBeforeScopeCheck, "mixed-purpose executed scope verification adds no model calls");
+    for (let index = 1; index < PIPELINE_STAGES.length; index += 1) {
+        const stage = PIPELINE_STAGES[index];
+        const parent = first.checkpoints[PIPELINE_STAGES[index - 1]];
+        assert.equal(first.checkpoints[stage].parentDigest, checkpointDigestV1(parent));
+    }
+    const accounting = await receivedCalls(repo.caseDirectory);
+    assert.equal(first.state.modelCalls, accounting.received.length);
+    assert.ok(accounting.received.every(({ operationId }) => !operationId.includes(":admit")), "admit adds no language-model receipt");
+    const callsAfterAdmission = scripted.calls.length;
+    const ledgerAfterAdmission = accounting.ledger.length;
+    const rerun = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(rerun.state.status, "admitted");
+    assert.equal(scripted.calls.length, callsAfterAdmission, "valid checkpoints skip every model role");
+    const afterRerun = await receivedCalls(repo.caseDirectory);
+    assert.equal(afterRerun.ledger.length, ledgerAfterAdmission);
+    assert.equal(rerun.state.modelCalls, first.state.modelCalls);
+    const incompatible = await runPipeline({
+        context: {
+            ...context,
+            dimension: { ...context.dimension, includes: [...context.dimension.includes, "changed"] }
+        },
+        transportLayer: scripted.layer
+    });
+    assert.equal(incompatible.state.status, "failed");
+    assert.match(incompatible.state.reason ?? "", /different pipeline identity/u);
+    assert.equal(scripted.calls.length, callsAfterAdmission, "an incompatible scientific identity is rejected before any model call");
+});
+test("pipeline resumes after an injected transport interruption", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp-interrupt", 80);
+    const scripted = transport({ interruptOnceAt: ":spec:writer:" });
+    const interrupted = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(interrupted.state.status, "running", interrupted.state.reason);
+    assert.equal(interrupted.state.stage, "spec");
+    assert.match(interrupted.state.reason ?? "", /injected transport interruption after retry/u);
+    assert.ok(interrupted.checkpoints.seed !== undefined, "accepted seed survives the interruption");
+    const seedDigest = checkpointDigestV1(interrupted.checkpoints.seed);
+    const callsBeforeResume = scripted.calls.length;
+    const resumed = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(resumed.state.status, "admitted");
+    assert.equal(resumed.admit?.verdict, "admitted");
+    assert.ok(resumed.state.resumeCount >= 1);
+    assert.ok(scripted.calls.length > callsBeforeResume);
+    assert.equal(checkpointDigestV1(resumed.checkpoints.seed), seedDigest);
+    assert.equal(scripted.calls.filter(({ operationId }) => operationId.includes(":pipeline:seed")).length, 0, "resume does not rerun the accepted seed checkpoint");
+});
+test("pipeline checkpoints each lazy critic round before a later interruption", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp-critic-resume", 80);
+    const scripted = transport({
+        oracleRejectsValidAlternative: true,
+        interruptOnceAt: "solution-critic-b"
+    });
+    const interrupted = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(interrupted.state.status, "running", interrupted.state.reason);
+    assert.equal(interrupted.state.stage, "tournament");
+    assert.equal(interrupted.checkpoints.tournament?.critiques?.filter(({ solutionId }) => solutionId === "solver-b").length, 1, "the accepted first critic round is durable before the second critic completes");
+    assert.equal(scripted.calls.filter(({ operationId }) => operationId.includes("solution-critic-a")).length, 1);
+    const resumed = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(resumed.state.status, "admitted", resumed.state.reason);
+    assert.equal(resumed.admit?.verdict, "admitted");
+    assert.equal(scripted.calls.filter(({ operationId }) => operationId.includes("solution-critic-a")).length, 1, "resume does not repeat an accepted checkpointed critic round");
+    assert.equal(resumed.checkpoints.tournament?.critiques?.filter(({ solutionId }) => solutionId === "solver-b")
+        .length, 2);
+});
+test("tournament repair accepts a mixed-purpose fixture for mapped preserved scope", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp-mixed-purpose-repair", 80);
+    const scripted = transport({
+        oracleRejectsValidAlternative: true,
+        mixedPurposeRepairFixture: true
+    });
+    const completed = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(completed.state.status, "admitted", completed.state.reason);
+    assert.equal(completed.admit?.verdict, "admitted");
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_tournament_repair_v1").length, 1, "a valid mixed-purpose replacement is accepted without a feedback retry");
+    assert.equal(completed.checkpoints.tournament?.suite.fixtures.find(({ id }) => id === "upper")
+        ?.expectationMode, "changes");
+});
+test("tournament repairs executed scope gaps before freeze or held-out work", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp-scope-repair", 80);
+    const scripted = transport({
+        oracleMissesChangedScope: true,
+        interruptOnceAt: ":pipeline:tournament:repair:"
+    });
+    const interrupted = await runPipeline({ context, transportLayer: scripted.layer });
+    const measured = interrupted.checkpoints.tournament;
+    assert.equal(measured.adequacy.admitted, false, "generic control separation cannot hide a missing grounded behavior");
+    assert.deepEqual(measured.adequacy.falseAcceptedSolutionIds, []);
+    assert.deepEqual(measured.adequacy.falseRejectedSolutionIds, []);
+    assert.match(measured.adequacy.rejectionReasons.join(";"), /changed scope lower-bound/u);
+    assert.equal(interrupted.checkpoints.challenge, undefined, "do not spend held-out work on a known repairable scope gap");
+    assert.equal(scripted.calls.some(({ schemaName }) => schemaName === "routekit_pipeline_held_out_adversary_v1"), false);
+    const resumed = await runPipeline({ context, transportLayer: scripted.layer });
+    assert.equal(resumed.state.status, "admitted", resumed.state.reason);
+    assert.equal(resumed.checkpoints.tournament?.history.at(-1)?.decision, "accepted", "reducing only an executed scope gap is scientific progress");
+    assert.equal(resumed.checkpoints.tournament?.adequacy.admitted, true);
+    const oracle = resumed.checkpoints.tournament.oracle;
+    const scopeInput = {
+        spec: resumed.checkpoints.spec, oracleCheckpoint: resumed.checkpoints.oracle,
+        oracle, repetitions: 2, label: "development"
+    };
+    assert.deepEqual(evaluateRepositoryOracleScopeCoverageV1(scopeInput).reasons, []);
+    for (const damage of ["missing", "duplicate", "unknown", "timeout", "unexecuted"]) {
+        const rows = oracle.observations.filter((row) => row.solutionId === "historical-defect" && row.clauseId === "fixture-negative");
+        assert.equal(rows.length, 2);
+        const observations = oracle.observations.filter((row) => !rows.includes(row));
+        observations.push(...(damage === "missing" ? rows.slice(0, 1)
+            : damage === "duplicate" ? [rows[0], rows[0]]
+                : rows.map((row) => ({
+                    ...row,
+                    outcome: damage === "unknown" ? "unknown" : row.outcome,
+                    detail: damage === "timeout" ? row.detail.replace("timeout=false", "timeout=true")
+                        : damage === "unexecuted" ? row.detail.replace("gradeExecuted=true", "gradeExecuted=false")
+                            : row.detail
+                }))));
+        assert.deepEqual(evaluateRepositoryOracleScopeCoverageV1({
+            ...scopeInput, oracle: { ...oracle, observations }
+        }).unresolvedScopeIds, ["lower-bound"], `${damage} evidence cannot close a scope gate`);
+    }
+});
+test("tournament challenger resumes complete replay units without replacing its incumbent early", async (t) => {
+    const repo = await repository(t);
+    const context = contextFor(repo, "pipeline-clamp-challenger-resume", 80);
+    const scripted = transport({ oracleRejectsValidAlternative: true });
+    const base = makeCaseCheckpointStoreV1({ caseDirectory: repo.caseDirectory });
+    const captured = [];
+    const store = {
+        ...base,
+        writeWorkCheckpoint: (name, value) => base.writeWorkCheckpoint(name, value).pipe(Effect.tap(() => Effect.gen(function* () {
+            if (!name.startsWith("pipeline-execution-tournament-") || captured.length > 0)
+                return;
+            const { oracle } = Schema.decodeUnknownSync(Schema.Struct({
+                oracle: RepositoryHiddenOracleV1
+            }))(value);
+            if (oracle.observations.length === 0)
+                return;
+            captured.push(oracle);
+            return yield* Effect.die("injected challenger slice interruption");
+        })))
+    };
+    const interrupted = await runPipeline({ context, transportLayer: scripted.layer, store });
+    assert.equal(interrupted.state.status, "running", interrupted.state.reason);
+    assert.match(interrupted.state.reason ?? "", /injected challenger slice interruption/u);
+    assert.equal(captured.length, 1);
+    assert.equal(interrupted.checkpoints.tournament?.adequacy.admitted, false, "partial challenger evidence cannot promote or overwrite the incumbent");
+    assert.deepEqual(interrupted.checkpoints.tournament?.adequacy.falseRejectedSolutionIds, ["solver-b"]);
+    const resumed = await runPipeline({ context, transportLayer: scripted.layer, store: base });
+    assert.equal(resumed.state.status, "admitted", resumed.state.reason);
+    assert.equal(scripted.calls.filter(({ schemaName }) => schemaName === "routekit_pipeline_tournament_repair_v1").length, 1, "the accepted repair proposal is not purchased again");
+    for (const row of captured[0].observations) {
+        assert.deepEqual(resumed.checkpoints.tournament.oracle.observations.find((candidate) => candidate.solutionId === row.solutionId &&
+            candidate.clauseId === row.clauseId &&
+            candidate.repetition === row.repetition), row, "completed challenger replay evidence survives byte-for-byte");
+    }
+});
+test("pipeline resumes budget exhaustion with a larger cumulative budget", async (t) => {
+    const repo = await repository(t);
+    const scripted = transport();
+    const exhausted = await runPipeline({
+        context: contextFor(repo, "pipeline-clamp-budget", 2),
+        transportLayer: scripted.layer
+    });
+    assert.equal(exhausted.state.status, "exhausted");
+    assert.equal(exhausted.state.modelCalls, 2);
+    const callsAtExhaustion = scripted.calls.length;
+    const accountingAtExhaustion = await receivedCalls(repo.caseDirectory);
+    assert.equal(accountingAtExhaustion.received.length, 2);
+    const completed = await runPipeline({
+        context: contextFor(repo, "pipeline-clamp-budget", 80),
+        transportLayer: scripted.layer
+    });
+    assert.equal(completed.state.status, "admitted", completed.state.reason);
+    assert.equal(completed.admit?.verdict, "admitted");
+    assert.ok(completed.state.resumeCount >= 1);
+    assert.ok(completed.state.modelCalls > exhausted.state.modelCalls);
+    assert.ok(scripted.calls.length > callsAtExhaustion);
+    const finalAccounting = await receivedCalls(repo.caseDirectory);
+    assert.equal(completed.state.modelCalls, finalAccounting.received.length);
+    assert.equal(finalAccounting.received.filter(({ operationId }) => operationId.includes(":admit")).length, 0);
+    const status = await Effect.runPromise(readCasePipelineStatusV1({ caseDirectory: repo.caseDirectory }));
+    assert.equal(status.state?.status, "admitted");
+    assert.equal(status.state?.modelCalls, completed.state.modelCalls);
+    assert.equal(status.ledgerEntries, finalAccounting.ledger.length);
+    assert.equal(status.checkpoints.admit?.verdict, "admitted");
+});