@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-oracle-coverage-witness.test.js

@@ -0,0 +1,511 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { REPOSITORY_FIXTURE_EVIDENCE_INSTRUCTIONS, repositoryFixtureAuthoringInstructionsV1 } from "../repository-fixture-instructions.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { assertRepositoryFoundryGenerationPlanV1, repositoryFoundryGenerationBudgetV1, repositoryFoundryGenerationPlanDigestV1 } from "../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { makeRepositoryFoundryEvidenceReconstructionV1, RepositoryFoundryEvidenceReconstruction } from "../services/evidence-reconstruction/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+import { resolveRepositoryOracleCoverageWitnessesV1, validateRepositoryOracleCoverageWitnessRevisionsV1 } from "../services/oracle-coverage-witness/service.js";
+const execute = promisify(execFile);
+const modelPlan = repositoryFoundryModelPlanV1({
+    primaryModel: "openai/gpt-6-astra",
+    oracleCriticModel: "openai/gpt-6-astra"
+});
+const referenceSource = "export const clamp = value => Math.min(100, Math.max(0, value));\n";
+const testSource = (assertion) => `import assert from "node:assert/strict";
+import test from "node:test";
+import { clamp } from "./subject.mjs";
+test("observable contract", () => { ${assertion} });
+`;
+const fractionTest = testSource("assert.equal(clamp(42.5), 42.5);");
+const integerTest = testSource("assert.equal(clamp(101), 100);");
+const proposal = (scopeId = "fractions") => ({
+    scopeId,
+    explanation: "Rounding in-range fractions violates exact value preservation.",
+    fileOverrides: [
+        {
+            path: "subject.mjs",
+            edits: [{ search: "Math.max(0, value)", replace: "Math.max(0, Math.round(value))" }]
+        }
+    ],
+    testPath: "case.test.mjs",
+    testSource: fractionTest,
+    expectedBehavior: "An in-range fraction remains exactly unchanged.",
+    expectationMode: "preserved"
+});
+const review = (ids = ["fractions"]) => ({
+    verdict: "revise",
+    detail: "Integer-only examples may conceal a fractional regression.",
+    scopeCoverage: ids.map((scopeId) => ({
+        scopeId,
+        outcome: "gap",
+        detail: "An isolated rounding defect may pass integer tests.",
+        counterexample: "Round an in-range fraction in an otherwise correct implementation.",
+        assertionEvidence: []
+    }))
+});
+async function fixture(t) {
+    const root = await realpath(await mkdtemp(join(tmpdir(), "coverage-witness-test-")));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const git = (...args) => execute("git", ["-C", root, ...args]);
+    await git("init", "-q");
+    await writeFile(join(root, "subject.mjs"), referenceSource);
+    await writeFile(join(root, "case.test.mjs"), integerTest);
+    await writeFile(join(root, "control.json"), "{}");
+    await git("add", ".");
+    await git("-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false", "-c", "user.name=Coverage Witness", "-c", "user.email=witness@example.test", "commit", "-qm", "pinned reference");
+    const commit = (await git("rev-parse", "HEAD")).stdout.trim();
+    const grade = {
+        id: "grade",
+        kind: "test",
+        cwd: ".",
+        executable: "node",
+        args: ["--test", "case.test.mjs"],
+        timeoutMs: 5_000
+    };
+    const observation = (id, outcome) => ({
+        id,
+        kind: "test-command",
+        subjectId: id === "baseline" ? "baseline" : "grade",
+        outcome,
+        detail: "Qualified test seed."
+    });
+    const seed = {
+        version: 1,
+        id: "witness-seed",
+        status: "qualified",
+        source: { kind: "historical-fix", changeEpisodeId: "change" },
+        summary: "Clamp bounds and preserve fractions.",
+        capabilityEvidence: [{ kind: "test", id: "case", path: "case.test.mjs" }],
+        initialState: { commit },
+        referenceCommit: commit,
+        targetBehavior: [
+            { id: "fractions", description: "Preserve in-range fractions.", critical: true, evidence: [] }
+        ],
+        baselineObservations: [observation("baseline", "pass")],
+        preChangeObservations: [observation("before", "fail")],
+        postChangeObservations: [observation("after", "pass")],
+        candidateTestIds: ["case"],
+        environment: {
+            protectedControlPaths: ["control.json"],
+            trustedPreparationRecipes: [],
+            candidateValidationRecipes: [
+                {
+                    id: "syntax",
+                    kind: "check",
+                    cwd: ".",
+                    executable: "node",
+                    args: ["--check", "subject.mjs"],
+                    timeoutMs: 5_000
+                }
+            ],
+            baselineGradeRecipes: [{ ...grade, id: "baseline" }],
+            gradeRecipes: [grade]
+        },
+        confidence: "high",
+        risks: [],
+        rejectionReasons: []
+    };
+    const suite = {
+        version: 1,
+        caseId: "witness-case",
+        fixtures: [
+            {
+                id: "upper",
+                kind: "counterfactual",
+                source: "generated",
+                description: "Cap the upper bound.",
+                expectedBehavior: "101 becomes 100.",
+                testPath: "case.test.mjs",
+                expectationMode: "changes"
+            }
+        ],
+        overlays: [{ path: "case.test.mjs", content: integerTest, fixtureIds: ["upper"] }]
+    };
+    // Pinned source and checkout isolation must win over ambient working-tree state.
+    await writeFile(join(root, "subject.mjs"), "throw new Error('dirty tree');");
+    return { root, seed, suite };
+}
+function runner(data, options = {}) {
+    const calls = [];
+    const artifacts = new Map();
+    const accounting = { attempts: 0 };
+    const effect = resolveRepositoryOracleCoverageWitnessesV1({
+        operationId: "witness-test",
+        witnessPrefix: "coverage-initial",
+        repositoryRoot: data.root,
+        seed: options.seed ?? data.seed,
+        suite: options.suite ?? data.suite,
+        review: options.review ?? review(),
+        modelPlan,
+        allowedTestPaths: new Set(["case.test.mjs"]),
+        allowedSolutionPaths: options.allowedSolutionPaths ?? new Set(["subject.mjs"]),
+        referenceSources: [{ path: "subject.mjs", content: referenceSource }],
+        context: { visible: "Preserve every in-range numeric value." },
+        maximumAttempts: options.maximumAttempts ?? 2,
+        oracleCoverageWitnessVersion: options.oracleCoverageWitnessVersion,
+        onGenerationAttempt: () => Effect.sync(() => {
+            accounting.attempts += 1;
+        })
+    }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive), Effect.provideService(EvalAuthoringTransport, options.transport ?? {
+        complete: () => Effect.die("detailed transport required"),
+        completeDetailed: (request) => Effect.sync(() => {
+            calls.push(request);
+            return {
+                text: JSON.stringify(options.reply?.(request, calls.length) ?? { witnesses: [proposal()] }),
+                callId: `witness-call-${String(calls.length)}`
+            };
+        })
+    }), Effect.provide(RepositoryFoundryProgressLive(() => Effect.void, (artifact) => Effect.sync(() => {
+        artifacts.set(artifact.operationId, artifact);
+    }))));
+    return { effect, calls, artifacts, accounting };
+}
+test("coverage witnesses prove an isolated escape and retain exact reconstructable replays", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data);
+    const result = await Effect.runPromise(run.effect);
+    assert.equal(createHash("sha256").update(run.calls[0].instructions).digest("hex"), "9f9cf7cb450c16ca3b5df037983ac2bf14770c37a9566ef7781dbeb6d04089c8", "V1 reconstructs the original recorded witness-authoring instruction bytes");
+    assert.equal(result.witnesses[0].outcome, "confirmed-escape");
+    assert.equal(result.witnesses[0].referenceEvidence.outcome, "pass");
+    assert.equal(result.witnesses[0].mutantEvidence.outcome, "fail");
+    assert.equal(result.suite.fixtures.length, 2);
+    assert.deepEqual(result.suite.overlays[0], data.suite.overlays[0]);
+    assert.equal(run.accounting.attempts, 1);
+    assert.equal("fixtureSuite" in JSON.parse(run.calls[0].input), false);
+    const replays = [...run.artifacts.values()].filter((artifact) => artifact.schemaName === "routekit_repository_coverage_witness_replay_v1");
+    assert.equal(replays.length, 3);
+    for (const artifact of replays) {
+        const request = artifact.request.input;
+        assert.equal(request.replayInput.repetitions, 2);
+        assert.equal(request.replayInput.commit, data.seed.referenceCommit);
+        assert.equal(artifact.value.results.length, 2);
+        assert.equal(artifact.value.admissionEvidence, false);
+    }
+    const unsigned = {
+        version: 1,
+        repository: {
+            root: data.root,
+            requestedRef: "HEAD",
+            commit: data.seed.referenceCommit,
+            tree: "tree"
+        },
+        caseId: data.suite.caseId,
+        oracleCoverageWitnessVersion: 1,
+        seedId: data.seed.id,
+        initialCommit: data.seed.initialState.commit,
+        referenceCommit: data.seed.referenceCommit,
+        modelPlan,
+        qualification: { batches: 2, repetitionsPerBatch: 2, oracleRepetitions: 2 },
+        budget: repositoryFoundryGenerationBudgetV1(2, undefined, 2, 2)
+    };
+    const plan = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+    assertRepositoryFoundryGenerationPlanV1(plan);
+    const artifacts = [...run.artifacts.values()].map((artifact, index) => ({
+        ...artifact,
+        planDigest: plan.planDigest,
+        caseId: plan.caseId,
+        updatedAt: new Date(index).toISOString()
+    }));
+    const saved = await Effect.runPromise(makeRepositoryFoundryEvidenceReconstructionV1({ plan, artifacts }));
+    const restored = await Effect.runPromise(runner(data, { transport: saved.transport }).effect.pipe(Effect.provideService(RepositoryFoundryEvidenceReconstruction, saved.service)));
+    await Effect.runPromise(saved.service.assertConsumed);
+    assert.deepEqual(restored, result);
+    const incomplete = await Effect.runPromise(makeRepositoryFoundryEvidenceReconstructionV1({
+        plan,
+        artifacts: artifacts.filter((artifact) => artifact !== artifacts.find((entry) => entry.operationId.includes(":incumbent:")))
+    }));
+    await assert.rejects(Effect.runPromise(runner(data, { transport: incomplete.transport }).effect.pipe(Effect.provideService(RepositoryFoundryEvidenceReconstruction, incomplete.service))), /infrastructure or incomplete execution/u);
+});
+test("already detected coverage criticism retains its isolated behavioral guard without claiming new coverage", async (t) => {
+    const data = await fixture(t);
+    const suite = {
+        ...data.suite,
+        overlays: [{ ...data.suite.overlays[0], content: fractionTest }]
+    };
+    const run = runner(data, { suite });
+    const result = await Effect.runPromise(run.effect);
+    assert.equal(result.witnesses[0].outcome, "already-detected");
+    assert.deepEqual(result.suite.overlays[0], suite.overlays[0]);
+    assert.equal(result.suite.fixtures.length, 2);
+    assert.equal(result.suite.overlays[1].content, fractionTest);
+});
+for (const defect of [
+    "bad-reference",
+    "runtime-mutant",
+    "syntax-mutant",
+    "multiple-tests"
+]) {
+    test(`invalid coverage witness is repaired without counting ${defect} as a semantic kill`, async (t) => {
+        const data = await fixture(t);
+        const run = runner(data, {
+            reply: (request, ordinal) => {
+                if (ordinal > 1) {
+                    assert.equal(JSON.parse(request.input).previousDiagnostics.length, 1);
+                    return { witnesses: [proposal()] };
+                }
+                const witness = proposal();
+                if (defect === "bad-reference")
+                    witness.testSource = testSource("assert.equal(clamp(42.5), 43);");
+                if (defect === "multiple-tests")
+                    witness.testSource += '\ntest("extra", () => assert.ok(true));';
+                if (defect === "runtime-mutant" || defect === "syntax-mutant") {
+                    witness.fileOverrides[0].edits[0].replace =
+                        defect === "runtime-mutant" ? "missingAtRuntime(value)" : "Math.max(0, ) +";
+                }
+                return { witnesses: [witness] };
+            }
+        });
+        const result = await Effect.runPromise(run.effect);
+        assert.equal(run.accounting.attempts, 2);
+        assert.equal(result.witnesses[0].outcome, "confirmed-escape");
+        const executions = [...run.artifacts.values()].filter((artifact) => artifact.schemaName === "routekit_repository_coverage_witness_execution_v1");
+        assert.equal(executions[0].value.resolved, false);
+        assert.equal(executions[1].value.resolved, true);
+    });
+}
+test("bounded witness repair retains resolved claims and requests only pending claims", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, {
+        review: review(["first", "second"]),
+        reply: (request, ordinal) => {
+            if (ordinal === 1)
+                return {
+                    witnesses: [
+                        proposal("first"),
+                        { ...proposal("second"), testSource: testSource("assert.fail('bad');") }
+                    ]
+                };
+            assert.deepEqual(JSON.parse(request.input).coverageHypotheses.map((claim) => claim.scopeId), ["second"]);
+            return { witnesses: [proposal("second")] };
+        }
+    });
+    const result = await Effect.runPromise(run.effect);
+    assert.deepEqual(result.witnesses.map((entry) => entry.scopeId), ["first", "second"]);
+    assert.equal(result.suite.fixtures.length, 3);
+    assert.equal([...run.artifacts.values()].filter((artifact) => artifact.operationId.endsWith("coverage-initial-1:reference")).length, 1);
+});
+test("invalid exact edits consume the bounded author allowance and never execute", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, {
+        reply: () => ({
+            witnesses: [
+                {
+                    ...proposal(),
+                    fileOverrides: [
+                        { path: "subject.mjs", edits: [{ search: "absent text", replace: "wrong" }] }
+                    ]
+                }
+            ]
+        })
+    });
+    await assert.rejects(Effect.runPromise(run.effect), /oracle-coverage-witness-exhausted/u);
+    assert.equal(run.accounting.attempts, 2);
+    assert.equal([...run.artifacts.values()].some((artifact) => artifact.schemaName === "routekit_repository_coverage_witness_replay_v1"), false);
+});
+test("protected paths and test edits cannot be used as witness implementations", async (t) => {
+    const data = await fixture(t);
+    for (const path of ["control.json", "case.test.mjs", "../subject.mjs"]) {
+        const run = runner(data, { allowedSolutionPaths: new Set([path]) });
+        await assert.rejects(Effect.runPromise(run.effect), /valid incumbent fixture suite/u);
+        assert.equal(run.accounting.attempts, 0);
+    }
+});
+test("infrastructure failures stop witness execution without a semantic repair retry", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, {
+        seed: {
+            ...data.seed,
+            environment: {
+                ...data.seed.environment,
+                trustedPreparationRecipes: [
+                    {
+                        id: "preparation",
+                        kind: "build",
+                        cwd: ".",
+                        executable: "node",
+                        args: ["-e", "process.exit(1)"],
+                        timeoutMs: 5_000
+                    }
+                ]
+            }
+        }
+    });
+    await assert.rejects(Effect.runPromise(run.effect), /infrastructure or incomplete execution/u);
+    assert.equal(run.accounting.attempts, 1);
+});
+const revisionEffect = (data, witness, suite, artifacts) => validateRepositoryOracleCoverageWitnessRevisionsV1({
+    operationId: "witness-test:revision",
+    repositoryRoot: data.root,
+    seed: data.seed,
+    suite,
+    retained: [{ witness, overlay: witness.overlay }]
+}).pipe(Effect.provide(RepositoryFoundryProgressLive(() => Effect.void, (artifact) => Effect.sync(() => {
+    artifacts.set(artifact.operationId, artifact);
+}))));
+test("V2 witness mechanics can change while the original mutant and complete replay lineage remain fixed", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, { oracleCoverageWitnessVersion: 2 });
+    const original = await Effect.runPromise(run.effect);
+    const witness = original.witnesses[0];
+    const revisedSource = testSource("for (const value of [0.25, 42.5, 99.75]) assert.equal(clamp(value), value);");
+    const suite = {
+        ...original.suite,
+        overlays: original.suite.overlays.map((overlay) => overlay.fixtureIds[0] === witness.fixture.id
+            ? { ...overlay, content: revisedSource }
+            : overlay)
+    };
+    const revisions = await Effect.runPromise(revisionEffect(data, witness, suite, run.artifacts));
+    assert.equal(revisions.length, 1);
+    assert.equal(revisions[0].status, "validated");
+    assert.equal(revisions[0].referenceEvidence.outcome, "pass");
+    assert.equal(revisions[0].mutantEvidence?.outcome, "fail");
+    assert.deepEqual(revisions[0].beforeOverlay, witness.overlay);
+    assert.equal(witness.overlay.content, fractionTest);
+    assert.equal(revisions[0].overlay.content, revisedSource);
+    assert.equal(revisions[0].originalExecutionOperationId, witness.executionOperationId);
+    assert.match(run.calls[0].instructions, /Transport chunks are not logical event boundaries/u);
+    assert.doesNotMatch(run.calls[0].instructions, /race against a resolved tagged sentinel/u);
+    const unsigned = {
+        version: 1,
+        repository: {
+            root: data.root,
+            requestedRef: "HEAD",
+            commit: data.seed.referenceCommit,
+            tree: "tree"
+        },
+        caseId: data.suite.caseId,
+        oracleCoverageWitnessVersion: 2,
+        seedId: data.seed.id,
+        initialCommit: data.seed.initialState.commit,
+        referenceCommit: data.seed.referenceCommit,
+        modelPlan,
+        qualification: { batches: 2, repetitionsPerBatch: 2, oracleRepetitions: 2 },
+        budget: repositoryFoundryGenerationBudgetV1(2, undefined, 2, 2)
+    };
+    const plan = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+    assertRepositoryFoundryGenerationPlanV1(plan);
+    const artifacts = [...run.artifacts.values()].map((artifact, index) => ({
+        ...artifact,
+        planDigest: plan.planDigest,
+        caseId: plan.caseId,
+        updatedAt: new Date(index).toISOString()
+    }));
+    const saved = await Effect.runPromise(makeRepositoryFoundryEvidenceReconstructionV1({ plan, artifacts }));
+    const restored = await Effect.runPromise(runner(data, {
+        oracleCoverageWitnessVersion: 2,
+        transport: saved.transport
+    }).effect.pipe(Effect.provideService(RepositoryFoundryEvidenceReconstruction, saved.service)));
+    const reconstructedRevisions = await Effect.runPromise(revisionEffect(data, restored.witnesses[0], suite, new Map()).pipe(Effect.provideService(RepositoryFoundryEvidenceReconstruction, saved.service)));
+    await Effect.runPromise(saved.service.assertConsumed);
+    assert.deepEqual(reconstructedRevisions, revisions);
+    const missing = await Effect.runPromise(makeRepositoryFoundryEvidenceReconstructionV1({
+        plan,
+        artifacts: artifacts.filter((artifact) => !(artifact.operationId.includes(":revision:") && artifact.operationId.endsWith(":mutant")))
+    }));
+    await assert.rejects(Effect.runPromise(revisionEffect(data, witness, suite, new Map()).pipe(Effect.provideService(RepositoryFoundryEvidenceReconstruction, missing.service))), /infrastructure or incomplete execution/u);
+});
+for (const defect of [
+    "lost-defect",
+    "rejects-reference",
+    "runtime-error",
+    "multiple-tests"
+]) {
+    test(`a rejected witness revision preserves the incumbent: ${defect}`, async (t) => {
+        const data = await fixture(t);
+        const run = runner(data);
+        const original = await Effect.runPromise(run.effect);
+        const witness = original.witnesses[0];
+        const source = defect === "lost-defect"
+            ? integerTest
+            : defect === "rejects-reference"
+                ? testSource("assert.equal(clamp(42.5), 43);")
+                : defect === "runtime-error"
+                    ? testSource("missingFunction();")
+                    : `${fractionTest}\ntest("second", () => assert.ok(true));`;
+        const suite = {
+            ...original.suite,
+            overlays: original.suite.overlays.map((overlay) => overlay.fixtureIds[0] === witness.fixture.id ? { ...overlay, content: source } : overlay)
+        };
+        const before = structuredClone(original);
+        const revisions = await Effect.runPromise(revisionEffect(data, witness, suite, run.artifacts));
+        assert.equal(revisions[0].status, "rejected");
+        assert.deepEqual(original, before);
+        if (defect === "lost-defect")
+            assert.equal(revisions[0].mutantEvidence?.outcome, "pass");
+        else
+            assert.equal(revisions[0].mutantEvidence, undefined);
+    });
+}
+test("witness revision cannot replace its reference, metadata, mutant path or test identity", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data);
+    const original = await Effect.runPromise(run.effect);
+    for (const witness of [
+        { ...original.witnesses[0], referenceCommit: "f".repeat(40) },
+        {
+            ...original.witnesses[0],
+            fixture: { ...original.witnesses[0].fixture, expectedBehavior: "weakened" }
+        },
+        { ...original.witnesses[0], fileOverrides: [{ path: "control.json", content: "{}" }] },
+        {
+            ...original.witnesses[0],
+            overlay: { ...original.witnesses[0].overlay, path: "another.test.mjs" }
+        }
+    ]) {
+        await assert.rejects(Effect.runPromise(revisionEffect(data, witness, original.suite, new Map())), /identity validation/u);
+    }
+});
+test("infrastructure failure during witness revision stops before adoption", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data);
+    const original = await Effect.runPromise(run.effect);
+    const witness = original.witnesses[0];
+    const suite = {
+        ...original.suite,
+        overlays: original.suite.overlays.map((overlay) => overlay.fixtureIds[0] === witness.fixture.id
+            ? { ...overlay, content: `${fractionTest}\n` }
+            : overlay)
+    };
+    const broken = {
+        ...data,
+        seed: {
+            ...data.seed,
+            environment: {
+                ...data.seed.environment,
+                trustedPreparationRecipes: [
+                    {
+                        id: "preparation",
+                        kind: "build",
+                        cwd: ".",
+                        executable: "node",
+                        args: ["-e", "process.exit(1)"],
+                        timeoutMs: 5_000
+                    }
+                ]
+            }
+        }
+    };
+    await assert.rejects(Effect.runPromise(revisionEffect(broken, witness, suite, new Map())), /infrastructure or incomplete execution/u);
+    assert.equal(witness.overlay.content, fractionTest);
+});
+test("V2 observation instructions preserve legacy prompt bytes and apply to complete repair prompts", () => {
+    const legacy = `prefix\n${REPOSITORY_FIXTURE_EVIDENCE_INSTRUCTIONS}\nsuffix`;
+    assert.equal(repositoryFixtureAuthoringInstructionsV1(legacy), legacy);
+    assert.equal(repositoryFixtureAuthoringInstructionsV1(legacy, 1), legacy);
+    const revised = repositoryFixtureAuthoringInstructionsV1(legacy, 2);
+    assert.match(revised, /^prefix\n/u);
+    assert.match(revised, /suffix$/u);
+    assert.match(revised, /fixed count of microtasks measures implementation scheduling/u);
+    assert.match(revised, /Accumulate reads to the required byte or event boundary/u);
+    assert.doesNotMatch(revised, /race against a resolved tagged sentinel/u);
+});

package/dist/test/repository-oracle-coverage.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-oracle-coverage.test.js

@@ -0,0 +1,168 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { assertRepositoryFoundryGenerationPlanV1, repositoryFoundryGenerationBudgetV1, repositoryFoundryGenerationPlanDigestV1, repositoryFoundryRoleOutputCeilingV1 } from "../repository-foundry-plan-protocol.js";
+import { assertRepositoryFoundryModelPlanV1, repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { assertRepositoryOracleCoverageReviewV1 } from "../repository-oracle-coverage-protocol.js";
+const primaryModel = "openai/gpt-6-astra";
+const content = "assert.equal(penalty(2000, 1000).until, 2000);";
+const suite = {
+    version: 1,
+    caseId: "deadline-preservation",
+    fixtures: [
+        {
+            id: "deadline",
+            kind: "metamorphic",
+            description: "An earlier reset preserves the later penalty deadline.",
+            expectedBehavior: "The supplied deadline remains unchanged.",
+            expectationMode: "preserved",
+            source: "generated",
+            testPath: "test.js"
+        }
+    ],
+    overlays: [{ path: "test.js", fixtureIds: ["deadline"], content }]
+};
+const approved = () => ({
+    verdict: "approve",
+    detail: "The conflicting reset is exercised while the supplied penalty remains authoritative.",
+    scopeCoverage: [
+        {
+            scopeId: "preserve-deadline",
+            outcome: "covered",
+            detail: "The reset occurs earlier than the supplied penalty deadline.",
+            counterexample: "",
+            assertionEvidence: [{ fixtureId: "deadline", excerpt: content }]
+        }
+    ]
+});
+test("oracle coverage approval requires every scope clause and exact current fixture evidence", () => {
+    const valid = approved();
+    assertRepositoryOracleCoverageReviewV1(valid, ["preserve-deadline"], suite);
+    for (const invalid of [
+        { ...valid, scopeCoverage: [] },
+        { ...valid, scopeCoverage: [...valid.scopeCoverage, ...valid.scopeCoverage] },
+        { ...valid, scopeCoverage: [{ ...valid.scopeCoverage[0], scopeId: "unknown" }] },
+        { ...valid, scopeCoverage: [{ ...valid.scopeCoverage[0], assertionEvidence: [] }] },
+        {
+            ...valid,
+            scopeCoverage: [{ ...valid.scopeCoverage[0], counterexample: "An unresolved case." }]
+        },
+        {
+            ...valid,
+            scopeCoverage: [
+                {
+                    ...valid.scopeCoverage[0],
+                    assertionEvidence: [{ fixtureId: "deadline", excerpt: "assert.equal(alwaysPass, true);" }]
+                }
+            ]
+        },
+        {
+            ...valid,
+            scopeCoverage: [
+                {
+                    ...valid.scopeCoverage[0],
+                    assertionEvidence: [{ fixtureId: "invented-fixture", excerpt: content }]
+                }
+            ]
+        }
+    ]) {
+        assert.throws(() => assertRepositoryOracleCoverageReviewV1(invalid, ["preserve-deadline"], suite), /every frozen scope clause/u);
+    }
+    assert.throws(() => assertRepositoryOracleCoverageReviewV1(valid, ["preserve-deadline"], {
+        ...suite,
+        overlays: [
+            { ...suite.overlays[0], content: "assert.equal(penalty(2000, 3000).until, 2000);" }
+        ]
+    }));
+});
+test("oracle coverage gaps carry actionable isolated counterexamples and cannot approve", () => {
+    const gap = {
+        verdict: "revise",
+        detail: "A compound candidate can be killed by missing metadata and conceal a deadline defect.",
+        scopeCoverage: [
+            {
+                scopeId: "preserve-deadline",
+                outcome: "gap",
+                detail: "No assertion isolates deadline preservation under an earlier reset.",
+                counterexample: "Use otherwise-correct metadata with reset 1000 and supplied deadline 2000; assert the effective deadline stays 2000.",
+                assertionEvidence: []
+            }
+        ]
+    };
+    assertRepositoryOracleCoverageReviewV1(gap, ["preserve-deadline"], suite);
+    assert.throws(() => assertRepositoryOracleCoverageReviewV1({ ...gap, verdict: "approve" }, ["preserve-deadline"], suite));
+    assert.throws(() => assertRepositoryOracleCoverageReviewV1({
+        ...gap,
+        scopeCoverage: [{ ...gap.scopeCoverage[0], counterexample: "" }]
+    }, ["preserve-deadline"], suite));
+});
+test("optional oracle criticism preserves legacy roles and reserves every review and repair call", () => {
+    const legacyModels = repositoryFoundryModelPlanV1({ primaryModel });
+    const models = repositoryFoundryModelPlanV1({ primaryModel, oracleCriticModel: primaryModel });
+    assert.equal(legacyModels.assignments.length, 17);
+    assert.deepEqual(models.assignments.slice(0, 17), legacyModels.assignments);
+    assert.equal(models.assignments[17].role, "oracle-critic");
+    assertRepositoryFoundryModelPlanV1(legacyModels);
+    assertRepositoryFoundryModelPlanV1(models);
+    assert.throws(() => assertRepositoryFoundryModelPlanV1({
+        ...models,
+        assignments: [...models.assignments, models.assignments[17]]
+    }));
+    const legacy = repositoryFoundryGenerationBudgetV1(2, undefined, 2);
+    for (const revisions of [0, 1, 2]) {
+        const budget = repositoryFoundryGenerationBudgetV1(2, undefined, 2, revisions);
+        assert.equal(budget.plannedModelCalls - legacy.plannedModelCalls, 2 + 2 * revisions);
+        assert.equal(budget.requiredTotalInputTokens - legacy.requiredTotalInputTokens, (2 + 2 * revisions) * 512_000);
+        assert.equal(budget.requiredTotalOutputTokens - legacy.requiredTotalOutputTokens, (2 + revisions) * repositoryFoundryRoleOutputCeilingV1("oracle-critic") +
+            revisions * repositoryFoundryRoleOutputCeilingV1("oracle-designer"));
+        const unsigned = {
+            version: 1,
+            repository: { root: "/repo", requestedRef: "HEAD", commit: "commit", tree: "tree" },
+            caseId: "coverage",
+            seedId: "seed",
+            initialCommit: "initial",
+            referenceCommit: "reference",
+            modelPlan: models,
+            qualification: { batches: 2, repetitionsPerBatch: 2, oracleRepetitions: 2 },
+            budget
+        };
+        const plan = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+        assertRepositoryFoundryGenerationPlanV1(plan);
+        if (revisions > 0) {
+            for (const witnessVersion of [1, 2]) {
+                const { version, repository, caseId, ...rest } = unsigned;
+                const versioned = {
+                    version,
+                    repository,
+                    caseId,
+                    oracleCoverageWitnessVersion: witnessVersion,
+                    ...rest
+                };
+                const witnessPlan = {
+                    ...versioned,
+                    planDigest: repositoryFoundryGenerationPlanDigestV1(versioned)
+                };
+                assertRepositoryFoundryGenerationPlanV1(witnessPlan);
+                assert.deepEqual(witnessPlan.budget, plan.budget, "witnesses reuse the reserved author allowance");
+                assert.notEqual(witnessPlan.planDigest, plan.planDigest);
+                assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+                    ...plan,
+                    oracleCoverageWitnessVersion: witnessVersion
+                }), /digest/u);
+            }
+        }
+        else {
+            assert.throws(() => assertRepositoryFoundryGenerationPlanV1({ ...plan, oracleCoverageWitnessVersion: 1 }), /nonzero coverage allowance/u);
+        }
+        assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+            ...plan,
+            budget: { ...budget, maximumOracleCoverageRevisions: revisions === 2 ? 0 : 2 }
+        }), /digest/u);
+        assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+            ...plan,
+            modelPlan: legacyModels
+        }), /assigned oracle critic/u);
+    }
+    for (const revisions of [-1, 3, 0.5, Number.NaN]) {
+        assert.throws(() => repositoryFoundryGenerationBudgetV1(2, undefined, 2, revisions));
+    }
+});