@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-semantic-calibration.test.js

@@ -0,0 +1,688 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { test } from "node:test";
+import { Deferred, Effect, Fiber, Layer, Schema } from "effect";
+import { EvalProjectAuthoringError, RepositoryFoundryError } from "../errors.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { assertRepositorySemanticCalibrationPlanV1, assertRepositorySemanticCalibrationReportV1, compileRepositorySemanticCalibrationPlanV1, compileRepositorySemanticCalibrationReportV1, executeRepositorySemanticCalibrationPlanV1, repositorySemanticCalibrationExampleDigestV1, repositorySemanticCalibrationInputDigestV1 } from "../repository-semantic-calibration.js";
+import { executeRepositorySemanticReviewV1, prepareRepositorySemanticReviewV1 } from "../services/case-generation/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+const hash = (text) => createHash("sha256").update(text).digest("hex");
+const copy = (value) => JSON.parse(JSON.stringify(value));
+const modelPlan = repositoryFoundryModelPlanV1({
+    primaryModel: "openai/gpt-6-astra",
+    criticBModel: "openai/gpt-5.6-terra"
+});
+/**
+ * Synthetic protocol-test data only. The checker stdout below is deliberately
+ * supplied test data, NOT evidence of a live independent development campaign.
+ */
+const fixture = (citationSource) => {
+    const source = { path: "src/clamp.js", content: "export const clamp = (value) => value;\n" };
+    const supportingSource = {
+        path: "src/clamp-finite.js",
+        content: "export const clampFinite = (value) => Math.min(10, Math.max(0, value));\n"
+    };
+    const supportingSourceProvided = citationSource !== undefined && citationSource !== "unprovided";
+    const input = {
+        requestedDimension: "numeric boundary preservation",
+        repositoryEnvironment: {
+            commit: "a".repeat(40),
+            sources: citationSource === "repository" ? [supportingSource] : [],
+            localImports: {
+                commit: "a".repeat(40),
+                sources: citationSource === "local-import" ? [supportingSource] : [],
+                omitted: [],
+                unresolved: []
+            }
+        },
+        visible: {
+            lane: "repository-agent",
+            id: "visible-clamp",
+            requestText: "Clamp finite values into [0,10] and preserve fractional values inside the interval.",
+            checkoutCommit: "a".repeat(40),
+            allowedTools: ["read", "edit", "test"],
+            constraints: ["Preserve the exported clamp function."]
+        },
+        targetBehavior: [
+            { id: "lower", description: "Negative values become zero.", critical: true, evidence: [] },
+            { id: "upper", description: "Values above ten become ten.", critical: true, evidence: [] },
+            {
+                id: "preserve",
+                description: "In-range values remain unchanged.",
+                critical: true,
+                evidence: []
+            }
+        ],
+        preChangeSources: [
+            source,
+            ...(citationSource === "pre-change" ? [supportingSource] : [])
+        ],
+        solutions: [
+            {
+                id: "control-a",
+                family: "comparison-left",
+                fileOverrides: [
+                    {
+                        path: source.path,
+                        content: citationSource === undefined
+                            ? "export const clamp = (value) => Math.min(10, Math.max(0, value));\n"
+                            : 'import { clampFinite } from "./clamp-finite.js";\n' +
+                                "export const clamp = (value) => clampFinite(value);\n"
+                    }
+                ]
+            },
+            {
+                id: "control-b",
+                family: "comparison-right",
+                fileOverrides: [
+                    {
+                        path: source.path,
+                        content: citationSource === undefined
+                            ? "export const clamp = (value) => Math.max(0, Math.min(10, value));\n"
+                            : 'import { clampFinite } from "./clamp-finite.js";\n' +
+                                "export function clamp(value) { return clampFinite(value); }\n"
+                    }
+                ]
+            },
+            {
+                id: "control-c",
+                family: "other",
+                fileOverrides: [
+                    { path: source.path, content: "export const clamp = (value) => Math.min(10, value);\n" }
+                ]
+            }
+        ],
+        comparisonSolutionIds: ["control-a", "control-b"]
+    };
+    const labels = [
+        {
+            solutionId: "control-a",
+            expectedClass: "valid",
+            rationale: "PRIVATE_LABEL_RATIONALE: exact finite clamp identity.",
+            violatedBehaviorIds: [],
+            evidenceIds: ["check-transcript"]
+        },
+        {
+            solutionId: "control-b",
+            expectedClass: "valid",
+            rationale: "PRIVATE_LABEL_RATIONALE: equivalent complete finite clamp identity.",
+            violatedBehaviorIds: [],
+            evidenceIds: ["check-transcript"]
+        },
+        {
+            solutionId: "control-c",
+            expectedClass: "wrong",
+            rationale: "PRIVATE_LABEL_RATIONALE: negative finite inputs remain negative.",
+            violatedBehaviorIds: ["lower"],
+            evidenceIds: ["check-transcript"]
+        }
+    ];
+    const checkerSource = "// SYNTHETIC_CHECKER_SOURCE_FOR_PROTOCOL_TESTS_ONLY\n";
+    const transcript = JSON.stringify({
+        version: 1,
+        purpose: "development-label-adjudication",
+        inputDigest: repositorySemanticCalibrationInputDigestV1(input),
+        checkerId: "synthetic-independent-checker",
+        checkerSourceDigest: hash(checkerSource),
+        checks: labels.map((label) => ({
+            solutionId: label.solutionId,
+            classification: label.expectedClass,
+            repetitions: 2,
+            confirmed: true,
+            violatedBehaviorIds: label.violatedBehaviorIds
+        }))
+    });
+    const body = {
+        id: "clamp-development",
+        split: "development",
+        repository: {
+            id: "synthetic-protocol-test-repository",
+            commit: "a".repeat(40),
+            sourceDigests: [source, ...(supportingSourceProvided ? [supportingSource] : [])].map((entry) => ({ path: entry.path, sha256: hash(entry.content) }))
+        },
+        input,
+        adjudication: {
+            kind: "independent-executable-development-adjudication",
+            reservation: "not-reserved-holdout",
+            adjudicatorIds: ["synthetic-independent-checker"],
+            generatorIdentities: ["synthetic-control-author"],
+            completedAt: "2026-09-08T00:00:00.000Z",
+            inputDigest: repositorySemanticCalibrationInputDigestV1(input),
+            evidence: [
+                {
+                    id: "check-transcript",
+                    kind: "executable-development-check",
+                    checkerSource,
+                    checkerSourceSha256: hash(checkerSource),
+                    content: transcript,
+                    sha256: hash(transcript)
+                }
+            ],
+            labels
+        }
+    };
+    const example = { ...body, exampleDigest: repositorySemanticCalibrationExampleDigestV1(body) };
+    const manifest = {
+        version: 1,
+        purpose: "development-semantic-calibration",
+        thresholds: {
+            minimumScenarios: 1,
+            minimumValidControls: 2,
+            minimumWrongControls: 1,
+            maximumFalseValidRate: 0,
+            maximumFalseWrongRate: 0,
+            maximumUncertaintyRate: 0,
+            maximumInstabilityRate: 0,
+            requireCompleteExecution: true
+        },
+        examples: [example]
+    };
+    return { input, example, manifest };
+};
+const compile = (manifest = fixture().manifest, validControlReviewVersion = 2, repetitions = 3) => compileRepositorySemanticCalibrationPlanV1({
+    manifest,
+    modelPlan,
+    validControlReviewVersion,
+    repetitions
+});
+const response = (request) => ({
+    reviews: [
+        {
+            solutionId: "control-a",
+            classification: "valid",
+            detail: "PRIVATE_CRITIC_RATIONALE: the lower and upper requirements are implemented.",
+            violatedBehaviorIds: []
+        },
+        {
+            solutionId: "control-b",
+            classification: "valid",
+            detail: "PRIVATE_CRITIC_RATIONALE: the equivalent guard preserves finite values.",
+            violatedBehaviorIds: []
+        },
+        {
+            solutionId: "control-c",
+            classification: "wrong",
+            detail: "PRIVATE_CRITIC_RATIONALE: negative values violate the lower bound.",
+            violatedBehaviorIds: ["lower"]
+        }
+    ],
+    ...(request.schemaName === "routekit_repository_solution_review_v1"
+        ? {}
+        : {
+            implementationIndependence: {
+                outcome: "correlated",
+                detail: "Both controls implement the same complete behavioral clamp.",
+                implementations: ["control-a", "control-b"].map((solutionId) => ({
+                    solutionId,
+                    mechanism: "finite lower/upper clamp",
+                    sourcePaths: ["src/clamp.js"]
+                })),
+                correlatedFeatures: ["same-behavioral-mechanism"]
+            }
+        })
+});
+const transportFixture = (reply = (request, ordinal) => Effect.succeed({
+    text: JSON.stringify(response(request)),
+    callId: `synthetic-call-${String(ordinal)}`
+})) => {
+    const calls = [];
+    const completeDetailed = (request) => Effect.suspend(() => {
+        calls.push(request);
+        return reply(request, calls.length);
+    });
+    const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        completeDetailed,
+        complete: (request) => completeDetailed(request).pipe(Effect.map((result) => result.text))
+    }));
+    return { calls, layer: RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport)) };
+};
+test("semantic calibration freezes source, independent check, policy, model, threshold and repetition identities", () => {
+    const original = fixture();
+    const first = compile(original.manifest);
+    assertRepositorySemanticCalibrationPlanV1(first);
+    assert.deepEqual(first, compile(original.manifest));
+    assert.equal(first.callCount, 6);
+    assert.equal(first.maximumOutputTokensPerCall, 32768);
+    assert.equal(first.requiredTotalOutputTokens, 6 * 32768);
+    assert.equal(first.admissionCredit, false);
+    assert.equal(Object.isFrozen(first.manifest.examples[0].input.solutions), true);
+    const changed = copy(original.manifest);
+    changed.thresholds.maximumUncertaintyRate = 0.1;
+    assert.notEqual(compile(changed).planDigest, first.planDigest);
+    assert.notEqual(compile(original.manifest, 1).planDigest, first.planDigest);
+    assert.notEqual(compile(original.manifest, 2, 2).planDigest, first.planDigest);
+    assert.equal(first.manifest.thresholds.maximumUncertaintyRate, 0);
+    const changedModel = compileRepositorySemanticCalibrationPlanV1({
+        manifest: original.manifest,
+        repetitions: 3,
+        validControlReviewVersion: 2,
+        modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-5.6-terra" })
+    });
+    assert.notEqual(changedModel.planDigest, first.planDigest);
+    const forged = copy(first);
+    forged.preparedCritics[0].instructionsDigest = "f".repeat(64);
+    assert.throws(() => assertRepositorySemanticCalibrationPlanV1(forged), /stale/u);
+});
+for (const scenario of [
+    { source: "repository", version: 2, contextOnly: false, accepted: true },
+    { source: "local-import", version: 2, contextOnly: false, accepted: true },
+    { source: "pre-change", version: 2, contextOnly: false, accepted: true },
+    { source: "unprovided", version: 2, contextOnly: false, accepted: false },
+    { source: "local-import", version: 2, contextOnly: true, accepted: false },
+    { source: "local-import", version: 1, contextOnly: false, accepted: false }
+]) {
+    test(`critic source citations require supplied bytes and a changed-file anchor: ${scenario.source}, v${String(scenario.version)}, context-only=${String(scenario.contextOnly)}`, async () => {
+        const plan = compile(fixture(scenario.source).manifest, scenario.version, 2);
+        const transport = transportFixture((request, ordinal) => {
+            const value = copy(response(request));
+            for (const implementation of value.implementationIndependence.implementations) {
+                implementation.sourcePaths = [
+                    ...(scenario.contextOnly ? [] : ["src/clamp.js"]),
+                    "src/clamp-finite.js"
+                ];
+            }
+            return Effect.succeed({
+                text: JSON.stringify(value),
+                callId: `source-citation-${String(ordinal)}`
+            });
+        });
+        const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+            plan,
+            operationId: `source-citation-${scenario.source}-${String(scenario.version)}`
+        }).pipe(Effect.provide(transport.layer)));
+        assert.equal(transport.calls.length, 4);
+        assert.equal(report.counts.executionFailureCalls, 0);
+        if (scenario.accepted) {
+            assert.equal(report.counts.malformedCalls, 0);
+            assert.equal(report.counts.validAsValid, 8);
+            assert.equal(report.counts.wrongAsWrong, 4);
+            assert.equal(report.counts.unscoredControls, 0);
+            assert.equal(report.counts.independenceCorrelated, 4);
+            assert.ok(report.observations.every((entry) => entry.outcome.kind === "classified"));
+        }
+        else {
+            assert.equal(report.counts.malformedCalls, 4);
+            assert.equal(report.counts.unscoredControls, 12);
+            assert.ok(report.observations.every((entry) => entry.outcome.kind === "malformed" && entry.outcome.reason === "independence-review"));
+        }
+    });
+}
+for (const defect of [
+    "held-out",
+    "source-digest",
+    "checker-source",
+    "check-transcript",
+    "unconfirmed-label",
+    "unrepeated-check",
+    "human-without-evidence",
+    "self-adjudication",
+    "stale-input",
+    "unknown-model-field",
+    "undersized-population"
+]) {
+    test(`semantic calibration rejects ${defect} before any model request`, () => {
+        const manifest = copy(fixture().manifest);
+        const example = manifest.examples[0];
+        if (defect === "held-out")
+            Object.assign(example, { split: "held-out" });
+        if (defect === "source-digest")
+            example.repository.sourceDigests[0].sha256 = "f".repeat(64);
+        const evidence = example.adjudication.evidence[0];
+        assert.equal(evidence.kind, "executable-development-check");
+        if (defect === "checker-source" && evidence.kind === "executable-development-check")
+            evidence.checkerSource += "tampered";
+        if (defect === "check-transcript")
+            evidence.sha256 = "f".repeat(64);
+        if (defect === "unconfirmed-label" || defect === "unrepeated-check") {
+            const check = JSON.parse(evidence.content);
+            if (defect === "unconfirmed-label")
+                check.checks[2].classification = "valid";
+            else
+                check.checks[0].repetitions = 1;
+            evidence.content = JSON.stringify(check);
+            evidence.sha256 = hash(evidence.content);
+        }
+        if (defect === "human-without-evidence")
+            example.adjudication.kind = "independent-human-development-adjudication";
+        if (defect === "self-adjudication")
+            example.adjudication.generatorIdentities.push(example.adjudication.adjudicatorIds[0]);
+        if (defect === "stale-input")
+            example.input.visible.id = "changed-visible-id";
+        if (defect === "unknown-model-field")
+            Object.assign(example.input, { expectedLabels: example.adjudication.labels });
+        if (defect === "undersized-population")
+            manifest.thresholds.minimumScenarios = 2;
+        example.exampleDigest = repositorySemanticCalibrationExampleDigestV1(example);
+        assert.throws(() => compile(manifest));
+    });
+}
+for (const version of [null, 1, 2]) {
+    test(`shared production critic preparation and execution is used for policy ${String(version)}`, async () => {
+        const { input } = fixture();
+        const prepared = prepareRepositorySemanticReviewV1({
+            role: "solution-critic-a",
+            modelContext: {
+                requestedDimension: input.requestedDimension,
+                repositoryEnvironment: input.repositoryEnvironment
+            },
+            visible: input.visible,
+            targetBehavior: input.targetBehavior,
+            preChangeSources: input.preChangeSources,
+            solutions: input.solutions.map((solution) => ({
+                ...solution,
+                expectedClass: "DO_NOT_SEND",
+                privateFixture: "DO_NOT_SEND"
+            })),
+            comparisonSolutionIds: input.comparisonSolutionIds,
+            ...(version === null ? {} : { validControlReviewVersion: version })
+        });
+        assert.equal(JSON.stringify(prepared.input).includes("DO_NOT_SEND"), false);
+        assert.equal(prepared.maximumOutputTokens, 32768);
+        assert.equal(prepared.instructions.includes("policy version 2"), version === 2);
+        assert.equal("implementationComparison" in prepared.input, version !== null);
+        const transport = transportFixture();
+        const result = await Effect.runPromise(executeRepositorySemanticReviewV1({
+            modelPlan,
+            operationId: "shared-critic-test",
+            prepared
+        }).pipe(Effect.provide(transport.layer)));
+        assert.equal(result.call.role, "solution-critic-a");
+        assert.equal(transport.calls.length, 1);
+        assert.equal(transport.calls[0].input, JSON.stringify(prepared.input));
+        assert.equal(transport.calls[0].instructions, prepared.instructions);
+        assert.equal(transport.calls[0].schemaName, prepared.schemaName);
+        assert.equal(Schema.decodeUnknownSync(prepared.outputSchema)(result.value).reviews.length, 3);
+    });
+}
+test("calibration runs both real critic paths without adjudication leakage or admission credit", async () => {
+    const plan = compile();
+    const transport = transportFixture();
+    const persisted = [];
+    const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan,
+        operationId: "development-calibration",
+        recordObservation: (observation) => Effect.sync(() => {
+            persisted.push(observation);
+        })
+    }).pipe(Effect.provide(transport.layer)));
+    assert.equal(transport.calls.length, 6);
+    assert.equal(persisted.length, 6);
+    assert.equal(report.observations.length, 6);
+    assert.deepEqual(new Set(transport.calls.map((call) => call.foundryRole)), new Set(["solution-critic-a", "solution-critic-b"]));
+    for (const call of transport.calls) {
+        assert.equal(/PRIVATE_LABEL_RATIONALE|SYNTHETIC_CHECKER_SOURCE|expectedClass|adjudication/u.test(call.input), false);
+        const binding = plan.preparedCritics.find((critic) => critic.role === call.foundryRole);
+        assert.equal(hash(call.input), binding.inputDigest);
+        assert.equal(hash(call.instructions), binding.instructionsDigest);
+    }
+    assert.deepEqual(report.denominators, {
+        scenarios: 1,
+        models: 2,
+        critics: 2,
+        repetitions: 3,
+        distinctControls: 3,
+        distinctValidControls: 2,
+        distinctWrongControls: 1,
+        scheduledCalls: 6,
+        scheduledControlClassifications: 18,
+        scheduledValidClassifications: 12,
+        scheduledWrongClassifications: 6
+    });
+    assert.equal(report.counts.validAsValid, 12);
+    assert.equal(report.counts.wrongAsWrong, 6);
+    assert.equal(report.counts.independenceCorrelated, 6);
+    assert.equal(report.counts.stableClassificationGroups, 6);
+    assert.equal(report.admissionCredit, false);
+    assert.equal(report.coverage.mechanismIndependence, "unadjudicated");
+    assert.equal(report.coverage.fullValidControlAcceptance, "not-established");
+    assert.equal("passed" in report, false);
+    assert.equal(/PRIVATE_LABEL_RATIONALE|PRIVATE_CRITIC_RATIONALE|export const clamp/u.test(JSON.stringify(report)), false);
+    assert.equal(report.controls.length, 3);
+    assert.ok(report.controls.every((control) => /^[a-f0-9]{64}$/u.test(control.controlDigest)));
+    assertRepositorySemanticCalibrationReportV1({ plan, report });
+});
+test("confusion, uncertainty, and label instability retain exact per-control observations", async () => {
+    const plan = compile();
+    const transport = transportFixture((request, ordinal) => {
+        const value = copy(response(request));
+        if (request.foundryRole === "solution-critic-a" && request.operationId.includes(":repeat-1:")) {
+            value.reviews[0].classification = "wrong";
+            value.reviews[0].violatedBehaviorIds = ["lower"];
+        }
+        if (request.foundryRole === "solution-critic-a" && request.operationId.includes(":repeat-2:"))
+            value.reviews[1].classification = "uncertain";
+        if (request.foundryRole === "solution-critic-b" && request.operationId.includes(":repeat-3:")) {
+            value.reviews[2].classification = "valid";
+            value.reviews[2].violatedBehaviorIds = [];
+        }
+        return Effect.succeed({ text: JSON.stringify(value), callId: `confusion-${String(ordinal)}` });
+    });
+    const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan,
+        operationId: "confusion-calibration"
+    }).pipe(Effect.provide(transport.layer)));
+    assert.equal(report.counts.validAsValid, 10);
+    assert.equal(report.counts.validAsWrong, 1);
+    assert.equal(report.counts.validAsUncertain, 1);
+    assert.equal(report.counts.wrongAsValid, 1);
+    assert.equal(report.counts.wrongAsWrong, 5);
+    assert.equal(report.counts.changedClassificationGroups, 3);
+    assert.equal(report.counts.stableClassificationGroups, 3);
+    assert.equal(report.counts.malformedCalls, 0);
+    assert.equal(report.counts.unscoredControls, 0);
+    assert.equal(report.observations[0].outcome.kind, "classified");
+    assert.deepEqual(report.thresholds, plan.manifest.thresholds);
+    assert.equal("passed" in report, false, "failed declared thresholds must not become a fake pass");
+});
+for (const defect of [
+    "invalid-json",
+    "schema",
+    "review-population",
+    "behavior-reference",
+    "independence-review"
+]) {
+    test(`malformed ${defect} is counted separately from model execution failure`, async () => {
+        const plan = compile();
+        const transport = transportFixture((request, ordinal) => {
+            const value = copy(response(request));
+            if (defect === "schema")
+                Object.assign(value.reviews[0], { classification: "maybe" });
+            if (defect === "review-population")
+                value.reviews[2].solutionId = "control-a";
+            if (defect === "behavior-reference")
+                value.reviews[2].violatedBehaviorIds = ["invented"];
+            if (defect === "independence-review")
+                value.implementationIndependence.correlatedFeatures.push("same-behavioral-mechanism");
+            return Effect.succeed({
+                text: defect === "invalid-json" ? "not-json" : JSON.stringify(value),
+                callId: `malformed-${String(ordinal)}`
+            });
+        });
+        const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+            plan,
+            operationId: "malformed-calibration"
+        }).pipe(Effect.provide(transport.layer)));
+        assert.equal(transport.calls.length, 6);
+        assert.equal(report.counts.malformedCalls, 6);
+        assert.equal(report.counts.executionFailureCalls, 0);
+        assert.equal(report.counts.unscoredControls, 18);
+        assert.equal(report.counts.incompleteClassificationGroups, 6);
+        assert.ok(report.observations.every((entry) => entry.outcome.kind === "malformed" && entry.outcome.reason === defect));
+    });
+}
+test("an execution failure settles its started pair, then records every remaining call as not-run", async () => {
+    const plan = compile();
+    const transport = transportFixture((request, ordinal) => request.foundryRole === "solution-critic-a"
+        ? Effect.fail(new EvalProjectAuthoringError({
+            operation: "authoring-evaluations",
+            detail: "not-json in transport error prose must NOT classify this as malformed"
+        }))
+        : Effect.succeed({
+            text: JSON.stringify(response(request)),
+            callId: `settled-${String(ordinal)}`
+        }));
+    const persisted = [];
+    const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan,
+        operationId: "failed-calibration",
+        recordObservation: (observation) => Effect.sync(() => {
+            persisted.push(observation);
+        })
+    }).pipe(Effect.provide(transport.layer)));
+    assert.equal(transport.calls.length, 2);
+    assert.equal(persisted.length, 6);
+    assert.equal(report.counts.executionFailureCalls, 1);
+    assert.equal(report.counts.malformedCalls, 0);
+    assert.equal(report.counts.classifiedCalls, 1);
+    assert.equal(report.counts.notRunCalls, 4);
+    assert.equal(report.counts.unscoredControls, 15);
+    assert.ok(report.observations.slice(2).every((entry) => entry.outcome.kind === "not-run"));
+    assertRepositorySemanticCalibrationReportV1({ plan, report });
+});
+test("each finished observation is persisted before its sibling needs to finish", async () => {
+    const firstRecorded = Deferred.makeUnsafe();
+    const plan = compile();
+    const transport = transportFixture((request, ordinal) => Effect.gen(function* () {
+        if (request.foundryRole === "solution-critic-b" && request.operationId.includes(":repeat-1:"))
+            yield* Deferred.await(firstRecorded);
+        return { text: JSON.stringify(response(request)), callId: `hook-${String(ordinal)}` };
+    }));
+    const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan,
+        operationId: "prospective-observation",
+        recordObservation: (observation) => observation.role === "solution-critic-a" && observation.repetition === 1
+            ? Deferred.succeed(firstRecorded, undefined).pipe(Effect.asVoid)
+            : Effect.void
+    }).pipe(Effect.provide(transport.layer), Effect.timeout("5 seconds")));
+    assert.equal(report.counts.classifiedCalls, 6);
+});
+test("required observation I/O failure propagates its original typed error, never a model outcome", async () => {
+    const requiredWriteFailure = new Error("required observation write failed");
+    const transport = transportFixture();
+    const result = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan: compile(),
+        operationId: "required-observation-io",
+        recordObservation: () => Effect.fail(requiredWriteFailure)
+    }).pipe(Effect.provide(transport.layer), Effect.match({ onFailure: (error) => error, onSuccess: () => undefined })));
+    assert.equal(result, requiredWriteFailure);
+    assert.ok(transport.calls.length <= 2);
+});
+for (const failedPhase of ["received", "validated"]) {
+    test(`required ${failedPhase} raw artifact persistence failure is fatal, not a counted model failure`, async () => {
+        const artifactWriteFailure = new RepositoryFoundryError({
+            operation: "execute-generation",
+            detail: "failed to persist private authoring evidence",
+            cause: new Error("synthetic required artifact write failure")
+        });
+        const transport = transportFixture();
+        const observations = [];
+        let reportReturned = false;
+        const result = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+            plan: compile(),
+            operationId: `required-artifact-${failedPhase}`,
+            recordObservation: (observation) => Effect.sync(() => {
+                observations.push(observation);
+            })
+        }).pipe(Effect.provide(Layer.mergeAll(transport.layer, RepositoryFoundryProgressLive(undefined, (artifact) => artifact.validation === failedPhase ? Effect.fail(artifactWriteFailure) : Effect.void))), Effect.match({
+            onFailure: (error) => error,
+            onSuccess: () => {
+                reportReturned = true;
+                return undefined;
+            }
+        })));
+        assert.equal(result, artifactWriteFailure);
+        assert.equal(reportReturned, false);
+        assert.equal(observations.length, 0, "required I/O must not become execution-failure evidence");
+        assert.ok(transport.calls.length >= 1 && transport.calls.length <= 2);
+    });
+}
+test("interruption preserves returned observations without manufacturing a completed report", async () => {
+    const siblingStarted = Deferred.makeUnsafe();
+    const firstRecorded = Deferred.makeUnsafe();
+    let siblingInterrupted = false;
+    let reportReturned = false;
+    const persisted = [];
+    const transport = transportFixture((request, ordinal) => Effect.gen(function* () {
+        if (request.foundryRole === "solution-critic-a") {
+            yield* Deferred.await(siblingStarted);
+            return { text: JSON.stringify(response(request)), callId: `partial-${String(ordinal)}` };
+        }
+        yield* Deferred.succeed(siblingStarted, undefined);
+        return yield* Effect.never.pipe(Effect.onInterrupt(() => Effect.sync(() => {
+            siblingInterrupted = true;
+        })));
+    }));
+    const fiber = Effect.runFork(executeRepositorySemanticCalibrationPlanV1({
+        plan: compile(),
+        operationId: "interrupted-calibration",
+        recordObservation: (observation) => Effect.gen(function* () {
+            persisted.push(observation);
+            yield* Deferred.succeed(firstRecorded, undefined);
+        })
+    }).pipe(Effect.provide(transport.layer), Effect.tap(() => Effect.sync(() => {
+        reportReturned = true;
+    }))));
+    try {
+        await Effect.runPromise(Deferred.await(firstRecorded).pipe(Effect.timeout("5 seconds")));
+    }
+    finally {
+        await Effect.runPromise(Fiber.interrupt(fiber));
+    }
+    assert.equal(reportReturned, false);
+    assert.equal(siblingInterrupted, true);
+    assert.equal(persisted.length, 1);
+    assert.equal(persisted[0].outcome.kind, "classified");
+});
+test("report reconstruction rejects omitted, duplicate, relabeled, wrong-policy and altered-count evidence", async () => {
+    const plan = compile();
+    const transport = transportFixture();
+    const report = await Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan,
+        operationId: "report-integrity"
+    }).pipe(Effect.provide(transport.layer)));
+    for (const defect of [
+        "missing",
+        "duplicate",
+        "model",
+        "policy",
+        "counts",
+        "expected-label"
+    ]) {
+        const altered = copy(report);
+        if (defect === "missing")
+            altered.observations.pop();
+        if (defect === "duplicate")
+            altered.observations[1] = altered.observations[0];
+        if (defect === "model")
+            altered.observations[0].model = "openai/other-model";
+        if (defect === "policy")
+            altered.observations[0].instructionsDigest = "f".repeat(64);
+        if (defect === "counts")
+            altered.counts.validAsWrong = 0;
+        if (defect === "counts")
+            altered.counts.validAsValid = 99;
+        if (defect === "expected-label")
+            altered.controls[0].expectedClass = "wrong";
+        assert.throws(() => assertRepositorySemanticCalibrationReportV1({ plan, report: altered }), defect);
+    }
+    const changedNotRun = copy(report.observations);
+    changedNotRun[0].outcome = { kind: "not-run", reason: "prior-execution-failure" };
+    assert.throws(() => compileRepositorySemanticCalibrationReportV1({
+        plan,
+        operationId: report.operationId,
+        observations: changedNotRun
+    }), /not-run/u);
+});
+test("a stale plan is rejected before touching the model transport", async () => {
+    const stale = copy(compile());
+    stale.repetitions += 1;
+    const transport = transportFixture();
+    await assert.rejects(Effect.runPromise(executeRepositorySemanticCalibrationPlanV1({
+        plan: stale,
+        operationId: "stale-calibration"
+    }).pipe(Effect.provide(transport.layer))), /plan failed validation/u);
+    assert.equal(transport.calls.length, 0);
+});