@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-seed-selection.test.js

@@ -0,0 +1,504 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { access, mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { repositoryCommandTestEvidenceV1 } from "../adapters/repository-command-evidence.js";
+import { rankRepositorySeedCandidatesV1 } from "../adapters/repository-seed-selection.js";
+import { assertRepositoryFoundryGenerationPlanV1 } from "../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { planHistoricalRepositoryCaseGenerationV1 } from "../services/foundry/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { buildRepositoryBehaviorMapV1 } from "../services/repository-model/service.js";
+const execFilePromise = promisify(execFile);
+const modelPlan = repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" });
+const episode = (commit, subject, paths, occurredAt = "2026-01-01T00:00:00Z") => ({
+    id: `git-${commit}`,
+    commit,
+    parentCommit: "parent",
+    occurredAt,
+    subject,
+    body: "",
+    changedPaths: paths.map((path) => ({
+        path,
+        status: "modified",
+        kind: path.includes("test") ? "test" : "implementation"
+    })),
+    testPaths: paths.filter((path) => path.includes("test")),
+    implementationPaths: paths.filter((path) => !path.includes("test")),
+    protocolPaths: [],
+    behaviorChanging: true,
+    seedEligible: true
+});
+test("seed ranking uses dimension evidence and deterministic narrow-change tie breaks", () => {
+    const narrow = episode("narrow", "fix wire protocol translation", [
+        "src/wire.ts",
+        "test/wire.test.ts"
+    ]);
+    const broad = episode("broad", "fix wire protocol translation", [
+        "src/wire.ts",
+        "src/other.ts",
+        "test/wire.test.ts"
+    ]);
+    const unrelated = episode("unrelated", "fix account usage", [
+        "src/accounts.ts",
+        "test/accounts.test.ts"
+    ]);
+    const docs = episode("docs", "fix wire protocol translation", [
+        "examples/wire.ts",
+        "examples/wire.test.ts"
+    ]);
+    const map = {
+        historyEpisodes: [unrelated, broad, docs, narrow],
+        tests: []
+    };
+    const ranked = rankRepositorySeedCandidatesV1(map, "wire-protocol-compatibility");
+    assert.deepEqual(ranked.map((entry) => entry.episode.commit), ["narrow", "broad", "unrelated"]);
+    assert.deepEqual(ranked[0]?.matchedTerms, ["wire", "protocol"]);
+    assert.equal(ranked[2]?.relevance, 0, "lack of lexical overlap is not a semantic exclusion");
+    assert.deepEqual(rankRepositorySeedCandidatesV1({ ...map, historyEpisodes: [...map.historyEpisodes].reverse() }, "wire-protocol-compatibility"), ranked);
+});
+test("bounded retrieval prefers concentrated path evidence over broad changes and subject-only keyword overlap", () => {
+    const narrow = episode("narrow", "fix buffered events", ["src/wire.ts", "test/wire.test.ts"]);
+    const subjectOnly = episode("subject-only", "fix wire protocol compatibility", [
+        "src/project.ts",
+        "src/authoring.ts",
+        "test/project.test.ts",
+        "test/authoring.test.ts"
+    ]);
+    const broad = episode("broad", "fix wire protocol compatibility", [
+        "src/wire.ts",
+        "test/wire.test.ts",
+        ...Array.from({ length: 18 }, (_, index) => `src/feature-${index}.ts`)
+    ]);
+    const unrelated = episode("unrelated", "fix account state", [
+        "src/account.ts",
+        "test/account.test.ts"
+    ]);
+    const map = {
+        historyEpisodes: [broad, subjectOnly, unrelated, narrow],
+        tests: []
+    };
+    const ranked = rankRepositorySeedCandidatesV1(map, "wire-protocol-compatibility");
+    assert.deepEqual(ranked.map(({ episode }) => episode.commit), ["narrow", "subject-only", "broad", "unrelated"]);
+    assert.ok(ranked[2].relevance > ranked[0].relevance, "broader lexical overlap must not consume the first bounded replay slot");
+    assert.deepEqual(rankRepositorySeedCandidatesV1({ ...map, historyEpisodes: [...map.historyEpisodes].reverse() }, "wire-protocol-compatibility"), ranked);
+});
+test("seed relevance ignores grammatical filler without turning retrieval into semantic admission", () => {
+    const wire = episode("wire", "fix protocol translation", ["src/wire.ts", "test/wire.test.ts"]);
+    const unrelated = episode("unrelated", "preserve the behavior and all of the values", [
+        "src/selection.ts",
+        "test/selection.test.ts"
+    ]);
+    const map = {
+        historyEpisodes: [unrelated, wire],
+        tests: [
+            {
+                path: "test/selection.test.ts",
+                names: ["preserves the values and all of their state when the context is absent"]
+            }
+        ]
+    };
+    const specific = rankRepositorySeedCandidatesV1(map, "wire protocol");
+    const grammatical = rankRepositorySeedCandidatesV1(map, "the wire and the protocol for all of their uses when there are any");
+    assert.equal(specific[0]?.episode.commit, "wire");
+    assert.equal(grammatical[0]?.episode.commit, "wire");
+    assert.ok(grammatical.every(({ matchedTerms }) => matchedTerms.every((term) => !["the", "and", "for", "all", "their", "when"].includes(term))));
+    const onlyGrammar = rankRepositorySeedCandidatesV1(map, "the and for all their when");
+    assert.equal(onlyGrammar.length, 2, "lexical no-match must not discard an eligible seed");
+    assert.ok(onlyGrammar.every(({ relevance, matchedTerms }) => relevance === 0 && matchedTerms.length === 0));
+    assert.deepEqual(rankRepositorySeedCandidatesV1({ ...map, historyEpisodes: [...map.historyEpisodes].reverse() }, "the and for all their when"), onlyGrammar, "empty content queries keep deterministic ordering without claiming target fit");
+});
+test("filler-only changes to dimension prose cannot manufacture retrieval evidence", () => {
+    const wire = episode("wire", "fix wire protocol", ["src/wire.ts", "test/wire.test.ts"]);
+    const unrelated = episode("unrelated", "the state and the values", [
+        "src/selection.ts",
+        "test/selection.test.ts"
+    ]);
+    const map = {
+        historyEpisodes: [wire, unrelated],
+        tests: []
+    };
+    assert.deepEqual(rankRepositorySeedCandidatesV1(map, "Wire protocol"), rankRepositorySeedCandidatesV1(map, "The wire and the protocol for all of their uses"), "function words must not contribute to subjects, paths, or test-name scores");
+});
+test("automatic seed retrieval excludes deleted or renamed test overlays without excluding ordinary production changes", () => {
+    const ordinary = episode("ordinary", "fix wire handling", ["src/wire.ts", "test/wire.test.ts"]);
+    const removedTest = {
+        ...ordinary,
+        commit: "deleted-test",
+        changedPaths: ordinary.changedPaths.map((entry) => entry.kind === "test" ? { ...entry, status: "deleted" } : entry)
+    };
+    const renamedTest = {
+        ...ordinary,
+        commit: "renamed-test",
+        changedPaths: ordinary.changedPaths.map((entry) => entry.kind === "test"
+            ? { ...entry, status: "renamed", previousPath: "test/old-wire.test.ts" }
+            : entry)
+    };
+    const renamedSource = {
+        ...ordinary,
+        commit: "renamed-source",
+        changedPaths: ordinary.changedPaths.map((entry) => entry.kind === "implementation"
+            ? { ...entry, status: "renamed", previousPath: "src/old-wire.ts" }
+            : entry)
+    };
+    const addedTest = {
+        ...ordinary,
+        commit: "added-test",
+        changedPaths: ordinary.changedPaths.map((entry) => entry.kind === "test" ? { ...entry, status: "added" } : entry)
+    };
+    const map = {
+        historyEpisodes: [removedTest, renamedTest, ordinary, renamedSource, addedTest],
+        tests: []
+    };
+    assert.deepEqual(rankRepositorySeedCandidatesV1(map, "wire")
+        .map(({ episode }) => episode.commit)
+        .sort(), ["added-test", "ordinary", "renamed-source"]);
+});
+test("incidental documentation changes remain eligible without admitting example implementation or test targets", () => {
+    const production = episode("production", "fix response handling", [
+        "src/wire.ts",
+        "test/wire.test.ts"
+    ]);
+    const documented = {
+        ...production,
+        commit: "documented",
+        changedPaths: [
+            ...production.changedPaths,
+            {
+                path: "docs/wire-protocol-compatibility.md",
+                status: "modified",
+                kind: "documentation"
+            }
+        ]
+    };
+    const toy = episode("toy", "fix wire protocol compatibility", [
+        "examples/wire.ts",
+        "examples/wire.test.ts"
+    ]);
+    const map = {
+        historyEpisodes: [production, documented, toy],
+        tests: []
+    };
+    const ranked = rankRepositorySeedCandidatesV1(map, "wire-protocol-compatibility");
+    assert.deepEqual(ranked.map(({ episode }) => episode.commit), ["documented", "production"]);
+    assert.equal(ranked[0].relevance, ranked[1].relevance, "incidental doc filenames cannot boost relevance");
+});
+test("automatic preparation requires a dimension and an explicit bounded candidate budget", async () => {
+    for (const input of [
+        {},
+        { requestedDimension: "normalizer", maximumSeedCandidates: 0 },
+        { requestedDimension: "normalizer", maximumSeedCandidates: 11 },
+        { requestedDimension: "normalizer", maximumSeedCandidates: 1.5 }
+    ]) {
+        await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+            repositoryRoot: "/not-inspected",
+            caseId: "case",
+            modelPlan,
+            ...input
+        })), /request failed validation/u);
+    }
+});
+test("automatic preparation rejects an inconclusive candidate then pins a separately indexed historical seed", async () => {
+    const root = await mkdtemp(join(tmpdir(), "routekit-seed-selection-"));
+    const git = async (...args) => (await execFilePromise("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
+    const commit = async (subject, date) => {
+        await git("add", ".");
+        await execFilePromise("git", ["-C", root, "commit", "-q", "-m", subject], {
+            env: { ...process.env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }
+        });
+        return git("rev-parse", "HEAD");
+    };
+    const implementation = join(root, "src/normalizer.js");
+    const regression = join(root, "test/normalizer.test.js");
+    const testSource = (body) => [
+        "import test from 'node:test'; import assert from 'node:assert/strict';",
+        "import { normalize } from '../src/normalizer.js';",
+        body,
+        ""
+    ].join("\n");
+    const manifest = {
+        name: "fixture-normalizer",
+        private: true,
+        type: "module",
+        scripts: { test: "node --test test/*.test.js" }
+    };
+    try {
+        await git("init", "-q");
+        await git("config", "user.name", "Foundry Test");
+        await git("config", "user.email", "foundry@example.test");
+        await mkdir(join(root, "src"));
+        await mkdir(join(root, "test"));
+        await writeFile(join(root, "package.json"), JSON.stringify(manifest));
+        await writeFile(implementation, "export const normalize = value => value;\n");
+        await writeFile(regression, testSource("test('normalizer preserves plain text', () => assert.equal(normalize('value'), 'value'));"));
+        const initial = await commit("initial normalizer", "2026-01-01T00:00:00Z");
+        await writeFile(implementation, "export const normalize = value => value.trim();\n");
+        await writeFile(regression, testSource("test('normalizer trims whitespace', () => assert.equal(normalize('  value  '), 'value'));"));
+        const good = await commit("fix(normalizer): trim whitespace", "2026-01-02T00:00:00Z");
+        await writeFile(implementation, "export const normalize = value => value.trim().toLowerCase();\n");
+        await writeFile(regression, testSource("test('normalizer folds case', () => { if (normalize(' VALUE ') !== 'value') throw new TypeError('fixture setup failure'); assert.equal(normalize(' VALUE '), 'value'); });"));
+        const bad = await commit("fix(normalizer): fold case", "2026-01-03T00:00:00Z");
+        // The current harness no longer runs either historical suite. Each selected
+        // episode must be reindexed at its own reference before environment inference.
+        await writeFile(join(root, "package.json"), JSON.stringify({ ...manifest, scripts: { test: "node --test unavailable/*.test.js" } }));
+        const tip = await commit("chore: move current test runner", "2026-01-04T00:00:00Z");
+        const events = [];
+        const progress = RepositoryFoundryProgressLive((event) => Effect.sync(() => {
+            events.push(event);
+        }));
+        const base = {
+            repositoryRoot: root,
+            requestedDimension: "normalizer",
+            caseId: "auto-normalizer",
+            validControlReviewVersion: 1,
+            modelPlan
+        };
+        await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({ ...base, maximumSeedCandidates: 1 })), /no historical seed qualified within 1 candidate attempts/u);
+        const plan = await Effect.runPromise(planHistoricalRepositoryCaseGenerationV1(base).pipe(Effect.provide(progress)));
+        assertRepositoryFoundryGenerationPlanV1(plan);
+        assert.equal(plan.referenceCommit, good);
+        assert.equal(plan.initialCommit, initial);
+        assert.equal(plan.repository.commit, good);
+        assert.equal(plan.requestedDimension, "normalizer");
+        assert.equal(plan.validControlReviewVersion, 1);
+        assert.ok(events.some((event) => event.detail?.includes(`tip=${tip}; candidates=2`)));
+        assert.ok(events.some((event) => event.detail?.includes(`candidate=${bad}; status=rejected`) &&
+            event.detail.includes("operational failure prevented grading")));
+        assert.ok(events.some((event) => event.detail === `seed-selection candidate=${good}; status=selected`));
+        assert.ok(events.every((event) => event.modelCalls === 0));
+        await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+            ...base,
+            referenceCommit: bad,
+            maximumSeedCandidates: 10
+        })), /operational failure prevented grading/u);
+        const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({ repositoryRoot: root, requestedRef: good }));
+        assert.equal(plan.repository.tree, map.repository.tree);
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});
+test("automatic selection retains real listener denial and skips pass/pass lexical distractors within its declared bound", {
+    skip: process.platform !== "darwin"
+}, async () => {
+    const root = await realpath(await mkdtemp(join(tmpdir(), "routekit-seed-selection-capability-")));
+    const canary = "private-composed-seed-qualification";
+    const git = async (...args) => (await execFilePromise("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
+    const commit = async (subject, date) => {
+        await git("add", ".");
+        await execFilePromise("git", [
+            "-C",
+            root,
+            "-c",
+            "core.hooksPath=/dev/null",
+            "-c",
+            "commit.gpgSign=false",
+            "-c",
+            "user.name=Foundry Test",
+            "-c",
+            "user.email=foundry@example.test",
+            "commit",
+            "-qm",
+            subject
+        ], { env: { ...process.env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } });
+        return git("rev-parse", "HEAD");
+    };
+    const testSource = (module, name, body) => [
+        "import test from 'node:test';",
+        "import assert from 'node:assert/strict';",
+        "import net from 'node:net';",
+        `import * as subject from '../src/${module}.js';`,
+        `test(${JSON.stringify(name)}, { timeout: 5_000 }, async () => {`,
+        `  console.log(${JSON.stringify(canary)});`,
+        "  console.log('diagnostic-cwd=' + process.cwd());",
+        body,
+        "});",
+        ""
+    ].join("\n");
+    const trimAssertion = "  assert.equal(subject.normalize('  value  '), 'value');";
+    try {
+        await git("init", "-q");
+        await mkdir(join(root, "src"));
+        await mkdir(join(root, "test"));
+        await writeFile(join(root, "package.json"), JSON.stringify({
+            name: "composed-seed-selection-fixture",
+            private: true,
+            type: "module",
+            scripts: { test: "node --test test/*.test.js" }
+        }));
+        await writeFile(join(root, "src/normalizer.js"), "export const normalize = value => value;\n");
+        await writeFile(join(root, "test/normalizer.test.js"), testSource("normalizer", "normalizer preserves plain text", "  assert.equal(subject.normalize('value'), 'value');"));
+        for (const side of ["left", "right"]) {
+            await writeFile(join(root, `src/${side}.js`), "export const value = 7;\n");
+            await writeFile(join(root, `test/${side}.test.js`), testSource(side, `unrelated ${side} counter`, "  assert.equal(subject.value, 7);"));
+        }
+        const initial = await commit("initial fixture", "2026-01-01T00:00:00Z");
+        await writeFile(join(root, "src/normalizer.js"), "export const normalize = value => value.trim();\n");
+        await writeFile(join(root, "test/normalizer.test.js"), testSource("normalizer", "normalizer trims whitespace", trimAssertion));
+        const good = await commit("fix: trim input", "2026-01-02T00:00:00Z");
+        const distractors = [];
+        for (const [index, side] of ["left", "right"].entries()) {
+            // Only the source comment and test name change. The module and assertion
+            // are unrelated to trimming, despite target words in the current test name.
+            await writeFile(join(root, `src/${side}.js`), `// Clarify the ${side} counter documentation.\nexport const value = 7;\n`);
+            await writeFile(join(root, `test/${side}.test.js`), testSource(side, `whitespace compatibility for unrelated ${side} counter`, "  assert.equal(subject.value, 7);"));
+            distractors.push(await commit(`chore: clarify ${side} counter`, `2026-01-0${String(index + 3)}T00:00:00Z`));
+        }
+        await writeFile(join(root, "src/normalizer.js"), "export const normalize = value => value.trim().toLowerCase();\n");
+        await writeFile(join(root, "test/normalizer.test.js"), testSource("normalizer", "normalizer handles whitespace after opening a listener", [
+            "  await new Promise((resolve, reject) => {",
+            "    const server = net.createServer();",
+            "    server.once('error', (error) => {",
+            "      console.error('listener-denied-code=' + error.code);",
+            // Preserve the original error. assert.fail would manufacture a
+            // semantic assertion failure from an unsupported runtime capability.
+            "      reject(error);",
+            "    });",
+            "    server.listen(0, '127.0.0.1', () => {",
+            "      server.close((error) => error === undefined ? resolve() : reject(error));",
+            "    });",
+            "  });",
+            "  assert.equal(subject.normalize('  VALUE  '), 'value');"
+        ].join("\n")));
+        const denied = await commit("fix: fold input case", "2026-01-05T00:00:00Z");
+        const expectedOrder = [denied, distractors[1], distractors[0], good];
+        const parents = new Map([
+            [good, initial],
+            [distractors[0], good],
+            [distractors[1], distractors[0]],
+            [denied, distractors[1]]
+        ]);
+        const discovery = await Effect.runPromise(buildRepositoryBehaviorMapV1({ repositoryRoot: root, requestedRef: "HEAD" }));
+        const ranked = rankRepositorySeedCandidatesV1(discovery, "whitespace");
+        assert.deepEqual(ranked.map(({ episode }) => episode.commit), expectedOrder);
+        assert.ok(ranked.every(({ matchedTerms }) => matchedTerms.includes("whitespace")));
+        // Equal-size changes share current test-name overlap; controlled dates
+        // establish this shortlist. Neither overlap nor rank certifies semantics.
+        const attempt = (maximumSeedCandidates) => {
+            const events = [];
+            const artifacts = [];
+            const progress = RepositoryFoundryProgressLive((event) => Effect.sync(() => {
+                events.push(event);
+            }), (artifact) => Effect.promise(async () => {
+                assert.equal(artifact.schemaName, "routekit_repository_seed_qualification_phase_v1");
+                const report = artifact.value;
+                // Inspect during recording, not after all selection is complete.
+                // Every isolated checkout must already have been released.
+                for (const result of report.results) {
+                    const checkout = /diagnostic-cwd=([^\r\n]+)/u.exec(result.stdout)?.[1];
+                    assert.ok(checkout, "the real grading command must expose its isolated cwd");
+                    await assert.rejects(access(checkout), { code: "ENOENT" });
+                }
+                artifacts.push(artifact);
+            }));
+            return {
+                events,
+                artifacts,
+                effect: planHistoricalRepositoryCaseGenerationV1({
+                    repositoryRoot: root,
+                    requestedDimension: "whitespace",
+                    caseId: `composed-seed-bound-${String(maximumSeedCandidates)}`,
+                    maximumSeedCandidates,
+                    qualificationBatches: 2,
+                    qualificationRepetitionsPerBatch: 2,
+                    modelPlan
+                }).pipe(Effect.provide(progress))
+            };
+        };
+        const exhausted = attempt(3);
+        await assert.rejects(Effect.runPromise(exhausted.effect), /no historical seed qualified within 3 candidate attempts/u);
+        const selected = attempt(4);
+        const plan = await Effect.runPromise(selected.effect);
+        assertRepositoryFoundryGenerationPlanV1(plan);
+        assert.equal(plan.initialCommit, initial);
+        assert.equal(plan.referenceCommit, good);
+        assert.equal(plan.repository.commit, good);
+        assert.equal(plan.repository.tree, await git("rev-parse", `${good}^{tree}`));
+        assert.equal(plan.requestedDimension, "whitespace");
+        for (const [run, references] of [
+            [exhausted, expectedOrder.slice(0, 3)],
+            [selected, expectedOrder]
+        ]) {
+            const details = run.events.flatMap(({ detail }) => (detail === undefined ? [] : [detail]));
+            assert.deepEqual(details.flatMap((detail) => {
+                const match = /^seed-selection candidate=([a-f0-9]+); attempt=\d+;/u.exec(detail);
+                return match === null ? [] : [match[1]];
+            }), references);
+            assert.deepEqual(details.flatMap((detail) => {
+                const match = /^seed-selection candidate=([a-f0-9]+); status=(rejected|selected)(?:;|$)/u.exec(detail);
+                return match === null ? [] : [{ commit: match[1], status: match[2] }];
+            }), references.map((reference) => ({
+                commit: reference,
+                status: reference === good ? "selected" : "rejected"
+            })));
+            const denialReason = details.find((detail) => detail.startsWith(`seed-selection candidate=${denied}; status=rejected;`));
+            assert.ok(denialReason);
+            assert.match(denialReason, /operational failure prevented grading/u);
+            assert.doesNotMatch(denialReason, /commands are unstable|cross-batch qualification diverged/u);
+            for (const distractor of distractors) {
+                const reason = details.find((detail) => detail.startsWith(`seed-selection candidate=${distractor}; status=rejected;`));
+                assert.ok(reason);
+                assert.match(reason, /every reviewed grade command must reproduce the pre-change failure/u);
+            }
+            assert.ok(run.events.length > 0);
+            assert.ok(run.events.every((event) => event.modelCalls === 0 && event.kind !== "model-call"), "even failed authoring starts would violate this local-only qualification baseline");
+            assert.doesNotMatch(JSON.stringify(run.events), /private-composed-seed-qualification|diagnostic-cwd|listener-denied-code|stdout|stderr/u);
+            assert.equal(run.artifacts.length, references.reduce((count, reference) => count + (reference === denied ? 2 : 6), 0));
+            assert.equal(new Set(run.artifacts.map((artifact) => artifact.operationId)).size, run.artifacts.length);
+            for (const reference of references) {
+                const artifacts = run.artifacts.filter((artifact) => artifact.value.referenceCommit === reference);
+                assert.equal(artifacts.length, reference === denied ? 2 : 6);
+                for (const batch of [1, 2]) {
+                    assert.deepEqual(artifacts
+                        .map((artifact) => artifact.value)
+                        .filter((report) => report.batch === batch)
+                        .map((report) => report.phase)
+                        .sort(), reference === denied
+                        ? batch === 1 ? ["baseline", "pre-change"] : []
+                        : ["baseline", "post-change", "pre-change"]);
+                }
+                for (const artifact of artifacts) {
+                    const report = artifact.value;
+                    assert.equal(artifact.response.text, JSON.stringify(report));
+                    assert.equal(artifact.role, "seed-qualifier");
+                    assert.equal(artifact.model, "local-replay");
+                    assert.equal(artifact.request.maximumOutputTokens, 0);
+                    assert.equal(report.diagnosticOnly, true);
+                    assert.equal(report.admissionEvidence, false);
+                    if (reference === good)
+                        assert.equal(report.seedId, plan.seedId);
+                    assert.deepEqual(report.results.map((result) => result.repetition), [1, 2]);
+                    for (const result of report.results) {
+                        const listenerDenied = reference === denied && report.phase !== "baseline";
+                        const expectedOutcome = listenerDenied
+                            ? "unknown"
+                            : reference === good && report.phase === "pre-change"
+                                ? "fail"
+                                : "pass";
+                        assert.equal(result.commit, report.phase === "post-change" ? reference : parents.get(reference));
+                        assert.equal(result.stage, "grade");
+                        assert.equal(result.gradeExecuted, true);
+                        assert.equal(result.timedOut, false);
+                        assert.equal(result.outputTruncated, false);
+                        assert.equal(result.exitCode, expectedOutcome === "pass" ? 0 : 1);
+                        assert.equal(repositoryCommandTestEvidenceV1(result, result.recipeId).outcome, expectedOutcome);
+                        assert.ok(result.stdout.includes(canary));
+                        if (listenerDenied) {
+                            const output = `${result.stdout}\n${result.stderr}`;
+                            assert.match(output, /listener-denied-code=(?:EPERM|EACCES)\b/u);
+                            assert.match(output, /code: ['"](?:EPERM|EACCES)['"]/u);
+                            assert.doesNotMatch(output, /ERR_ASSERTION|AssertionError/u);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});

package/dist/test/repository-semantic-calibration.test.d.ts

@@ -0,0 +1 @@
+export {};