@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-oracle-evidence.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-oracle-evidence.test.js

@@ -0,0 +1,185 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import { repositoryCommandProbeOutcomeV1, repositoryCommandTestEvidenceV1 } from "../adapters/repository-command-evidence.js";
+const runNodeTests = (testPath, reporter) => new Promise((resolve) => {
+    execFile(process.execPath, ["--test", `--test-reporter=${reporter}`, testPath], {
+        encoding: "utf8",
+        env: { ...process.env, NODE_TEST_CONTEXT: undefined },
+        timeout: 10_000
+    }, (error, stdout, stderr) => resolve({
+        recipeId: "reviewed-node-tests",
+        commit: "fixture",
+        repetition: 1,
+        stage: "grade",
+        exitCode: error === null ? 0 : typeof error.code === "number" ? error.code : null,
+        timedOut: error?.killed ?? false,
+        outputTruncated: false,
+        stdout,
+        stderr,
+        durationMs: 0,
+        gradeExecuted: true,
+        trustedPreparationResults: [],
+        candidateValidationResults: []
+    }));
+});
+test("probes never turn incomplete or unattributable execution into scientific outcomes", () => {
+    const result = {
+        recipeId: "grade",
+        commit: "fixture",
+        repetition: 1,
+        stage: "grade",
+        exitCode: 0,
+        timedOut: false,
+        outputTruncated: false,
+        stdout: "TAP version 13\n# Subtest: contract\nok 1 - contract\n# tests 1\n# pass 1\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n",
+        stderr: "",
+        durationMs: 1,
+        gradeExecuted: true,
+        trustedPreparationResults: [],
+        candidateValidationResults: []
+    };
+    assert.equal(repositoryCommandProbeOutcomeV1([result]), "completed");
+    assert.equal(repositoryCommandProbeOutcomeV1([]), "environment_blocked");
+    for (const incomplete of [
+        { timedOut: true },
+        { exitCode: null },
+        { outputTruncated: true },
+        { stdout: "" },
+        { exitCode: 1, stdout: "" },
+        { gradeExecuted: false }
+    ]) {
+        assert.equal(repositoryCommandProbeOutcomeV1([{ ...result, ...incomplete }]), "environment_blocked");
+    }
+    for (const stage of ["trustedPreparationResults", "candidateValidationResults"]) {
+        for (const incomplete of [{ exitCode: 2 }, { timedOut: true }, { outputTruncated: true }]) {
+            const blocked = { ...result, [stage]: [{ ...result, ...incomplete }] };
+            assert.equal(repositoryCommandTestEvidenceV1(blocked, result.recipeId).outcome, "unknown");
+            assert.equal(repositoryCommandProbeOutcomeV1([result, blocked]), "environment_blocked");
+        }
+    }
+    const assertion = {
+        ...result,
+        exitCode: 1,
+        stdout: "TAP version 13\n# Subtest: contract\nnot ok 1 - contract\n  failureType: 'testCodeFailure'\n  name: 'AssertionError'\n  code: 'ERR_ASSERTION'\n# tests 1\n# pass 0\n# fail 1\n# cancelled 0\n# skipped 0\n# todo 0\n"
+    };
+    assert.equal(repositoryCommandProbeOutcomeV1([assertion]), "scientific_rejection");
+});
+test("oracle evidence distinguishes real Node assertion results from execution and discovery failures", async (t) => {
+    const root = await mkdtemp(path.join(tmpdir(), "routekit-oracle-evidence-"));
+    const cases = [
+        {
+            name: "expected-exceptions",
+            source: "test('specified exceptions', async () => { assert.throws(() => { throw new TypeError('expected'); }, TypeError); await assert.rejects(async () => { throw new Error('expected'); }); });",
+            outcome: "pass"
+        },
+        {
+            name: "assertion-failure",
+            source: "test('public behavior', () => assert.equal(1, 2));",
+            outcome: "fail"
+        },
+        {
+            name: "runtime-failure",
+            source: "test('public behavior', () => { throw new TypeError('unrelated setup failure'); });",
+            outcome: "unknown"
+        },
+        {
+            name: "plain-error-deadline",
+            source: "test('streaming return deadline', async () => { await Promise.race([new Promise(() => {}), new Promise((_resolve, reject) => setTimeout(() => reject(new Error('stream did not return')), 10))]); });",
+            outcome: "unknown"
+        },
+        {
+            name: "asserted-deadline",
+            source: "test('streaming return deadline', async () => { const result = await Promise.race([new Promise(() => {}), new Promise(resolve => setTimeout(() => resolve('deadline'), 10))]); assert.notEqual(result, 'deadline', 'stream did not return'); });",
+            outcome: "fail"
+        },
+        {
+            name: "worker-exit",
+            source: "test('worker exits without behavioral evidence', () => process.exit(1));",
+            outcome: "unknown"
+        },
+        {
+            name: "runner-timeout",
+            source: "test('runner deadline', { timeout: 10 }, () => new Promise(() => {}));",
+            outcome: "unknown"
+        },
+        {
+            name: "nested-assertion-failure",
+            source: "test('outer contract', async (t) => { await t.test('inner behavior', () => assert.equal(1, 2)); });",
+            outcome: "fail"
+        },
+        {
+            name: "nested-runtime-failure",
+            source: "test('outer contract', async (t) => { await t.test('inner behavior', () => { throw new TypeError('broken setup'); }); });",
+            outcome: "unknown"
+        },
+        {
+            name: "runtime-error-with-assertion-code",
+            source: "test('public behavior', () => { throw Object.assign(new TypeError('broken setup'), { code: 'ERR_ASSERTION' }); });",
+            outcome: "unknown"
+        },
+        {
+            name: "mixed-failures",
+            source: "test('public behavior', () => assert.equal(1, 2)); test('broken setup', () => { throw new Error('setup failed'); });",
+            outcome: "unknown"
+        },
+        ...["beforeEach", "afterEach", "before", "after"].map((hook) => ({
+            name: `${hook}-assertion`,
+            source: `${hook}(() => assert.equal(1, 2)); test('public behavior', () => assert.equal(2, 2));`,
+            outcome: "unknown"
+        })),
+        { name: "syntax-failure", source: "export const broken = ;", outcome: "unknown" },
+        { name: "missing-import", source: "import './does-not-exist.mjs';", outcome: "unknown" },
+        { name: "empty-file", source: "", outcome: "unknown" },
+        {
+            name: "all-skipped",
+            source: "test.skip('public behavior', () => assert.equal(1, 2));",
+            outcome: "unknown"
+        }
+    ];
+    try {
+        for (const reporter of ["tap", "spec"]) {
+            for (const scenario of cases) {
+                await t.test(`${reporter}: ${scenario.name}`, async () => {
+                    const testPath = path.join(root, `${scenario.name}.test.mjs`);
+                    await writeFile(testPath, `import test, { beforeEach, afterEach, before, after } from 'node:test';\nimport assert from 'node:assert/strict';\n${scenario.source}\n`);
+                    const result = await runNodeTests(testPath, reporter);
+                    const evidence = repositoryCommandTestEvidenceV1(result, result.recipeId);
+                    assert.equal(evidence.outcome, reporter === "spec" && scenario.outcome === "fail" ? "unknown" : scenario.outcome, JSON.stringify({ evidence, stdout: result.stdout, stderr: result.stderr }));
+                    if (scenario.name === "plain-error-deadline") {
+                        assert.equal(evidence.inconclusiveReason, reporter === "tap" ? "unattributed-test-failure" : undefined);
+                    }
+                    if (scenario.name === "worker-exit" ||
+                        scenario.name === "runner-timeout" ||
+                        /^(?:before|after)/u.test(scenario.name)) {
+                        assert.equal(evidence.inconclusiveReason, undefined);
+                    }
+                    if (evidence.outcome === "pass") {
+                        for (const interruption of [
+                            { timedOut: true },
+                            { outputTruncated: true },
+                            { gradeExecuted: false, stage: "candidate-validation" },
+                            { exitCode: null },
+                            { recipeId: "different-recipe" },
+                            {
+                                candidateValidationResults: [
+                                    { ...result, stage: "candidate-validation", exitCode: 1 }
+                                ]
+                            }
+                        ]) {
+                            assert.equal(repositoryCommandTestEvidenceV1({ ...result, ...interruption }, result.recipeId)
+                                .outcome, "unknown");
+                        }
+                    }
+                });
+            }
+        }
+        assert.equal(repositoryCommandTestEvidenceV1(undefined, "missing").outcome, "unknown");
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});

package/dist/test/repository-oracle-plan.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-oracle-plan.test.js

@@ -0,0 +1,176 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { Effect } from "effect";
+import { assertRepositoryFoundryGenerationPlanV1, repositoryFoundryGenerationBudgetV1, repositoryFoundryGenerationPlanDigestV1 } from "../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { planHistoricalRepositoryCaseGenerationV1 } from "../services/foundry/service.js";
+const plan = (oracleConcurrency) => {
+    const unsigned = {
+        version: 1,
+        repository: { root: "/repo", requestedRef: "HEAD", commit: "commit", tree: "tree" },
+        caseId: "legacy-case",
+        seedId: "seed",
+        initialCommit: "initial",
+        referenceCommit: "reference",
+        modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" }),
+        qualification: {
+            batches: 2,
+            repetitionsPerBatch: 2,
+            oracleRepetitions: 2,
+            ...(oracleConcurrency === undefined ? {} : { oracleConcurrency })
+        },
+        budget: repositoryFoundryGenerationBudgetV1(2)
+    };
+    return { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+};
+test("oracle concurrency is digest-bound without changing legacy plans, repetitions, or model budgets", () => {
+    const legacy = plan();
+    assert.equal(legacy.planDigest, "357e58c2853e4379791e3d1c5b9475993a65c371684c8f6a6f4c69400f1d2994");
+    assert.equal(Object.hasOwn(legacy.qualification, "oracleConcurrency"), false);
+    assertRepositoryFoundryGenerationPlanV1(legacy);
+    const digests = new Set([legacy.planDigest]);
+    for (const concurrency of [1, 2, 3, 4]) {
+        const current = plan(concurrency);
+        assertRepositoryFoundryGenerationPlanV1(current);
+        assert.deepEqual(current.budget, legacy.budget);
+        assert.equal(current.qualification.oracleRepetitions, legacy.qualification.oracleRepetitions);
+        assert.equal(current.qualification.batches, legacy.qualification.batches);
+        assert.equal(current.qualification.repetitionsPerBatch, legacy.qualification.repetitionsPerBatch);
+        assert.equal(digests.has(current.planDigest), false);
+        digests.add(current.planDigest);
+        assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+            ...current,
+            qualification: { ...current.qualification, oracleConcurrency: concurrency === 4 ? 1 : 4 }
+        }), /digest/u);
+    }
+});
+test("invalid oracle concurrency fails before repository discovery and cannot enter a signed plan", async () => {
+    for (const oracleConcurrency of [0, -1, 0.5, 5, Number.NaN, Number.POSITIVE_INFINITY]) {
+        assert.throws(() => assertRepositoryFoundryGenerationPlanV1(plan(oracleConcurrency)));
+        await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+            repositoryRoot: "/does-not-exist-oracle-concurrency-regression",
+            referenceCommit: "reference",
+            caseId: "case",
+            oracleConcurrency,
+            modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" })
+        })), /repository foundry generation request failed validation/u);
+    }
+});
+test("coverage witness repair allocation is pinned and cannot create an unreserved call allowance", () => {
+    const { planDigest: _digest, ...base } = plan();
+    const unsignedPlan = (oracleCoverageWitnessRepairVersion) => ({
+        version: base.version,
+        repository: base.repository,
+        caseId: base.caseId,
+        oracleCoverageWitnessVersion: 2,
+        ...(oracleCoverageWitnessRepairVersion === undefined
+            ? {}
+            : { oracleCoverageWitnessRepairVersion }),
+        seedId: base.seedId,
+        initialCommit: base.initialCommit,
+        referenceCommit: base.referenceCommit,
+        modelPlan: repositoryFoundryModelPlanV1({
+            primaryModel: "openai/gpt-6-astra",
+            oracleCriticModel: "openai/gpt-6-astra"
+        }),
+        qualification: base.qualification,
+        budget: repositoryFoundryGenerationBudgetV1(2, undefined, 2, 2, 2)
+    });
+    const unsigned = unsignedPlan();
+    const legacy = {
+        ...unsigned,
+        planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned)
+    };
+    assertRepositoryFoundryGenerationPlanV1(legacy);
+    const revisedUnsigned = unsignedPlan(1);
+    const revised = {
+        ...revisedUnsigned,
+        planDigest: repositoryFoundryGenerationPlanDigestV1(revisedUnsigned)
+    };
+    assertRepositoryFoundryGenerationPlanV1(revised);
+    assert.notEqual(revised.planDigest, legacy.planDigest);
+    assert.deepEqual(revised.budget, legacy.budget);
+    assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+        ...legacy,
+        oracleCoverageWitnessRepairVersion: 1
+    }), /digest/u);
+    const { oracleCoverageWitnessVersion: _witnessVersion, ...withoutWitnesses } = revisedUnsigned;
+    const withoutRepairAllowance = {
+        ...revisedUnsigned,
+        budget: repositoryFoundryGenerationBudgetV1(2, undefined, 2, 2)
+    };
+    for (const invalid of [withoutWitnesses, withoutRepairAllowance]) {
+        assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+            ...invalid,
+            planDigest: repositoryFoundryGenerationPlanDigestV1(invalid)
+        }), /versioned witnesses and an explicit fixture repair allowance/u);
+    }
+});
+test("dependency context policy is immutable for both historical and API-aware plans", () => {
+    const { planDigest: _digest, ...base } = plan();
+    const digests = new Set();
+    for (const dependencyContextVersion of [undefined, 1, 2]) {
+        const unsigned = {
+            version: base.version,
+            repository: base.repository,
+            caseId: base.caseId,
+            ...(dependencyContextVersion === undefined ? {} : { dependencyContextVersion }),
+            seedId: base.seedId,
+            initialCommit: base.initialCommit,
+            referenceCommit: base.referenceCommit,
+            modelPlan: base.modelPlan,
+            qualification: base.qualification,
+            budget: base.budget
+        };
+        const signed = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+        assertRepositoryFoundryGenerationPlanV1(signed);
+        assert.equal(digests.has(signed.planDigest), false);
+        digests.add(signed.planDigest);
+        assert.deepEqual(signed.budget, base.budget);
+        for (const replacement of [1, 2, 3]) {
+            if (replacement === dependencyContextVersion)
+                continue;
+            assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+                ...signed,
+                dependencyContextVersion: replacement
+            }));
+        }
+        if (dependencyContextVersion !== undefined) {
+            const { dependencyContextVersion: _contextVersion, ...stripped } = signed;
+            assert.throws(() => assertRepositoryFoundryGenerationPlanV1(stripped), /digest/u);
+        }
+    }
+});
+test("repair feedback is digest-bound without changing the legacy plan or call allowance", async () => {
+    const legacy = plan();
+    const { planDigest: _digest, ...base } = legacy;
+    const { version, repository, caseId, ...remainder } = base;
+    const unsigned = {
+        version,
+        repository,
+        caseId,
+        oracleRepairFeedbackVersion: 1,
+        ...remainder
+    };
+    const current = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
+    assertRepositoryFoundryGenerationPlanV1(current);
+    assert.notEqual(current.planDigest, legacy.planDigest);
+    assert.deepEqual(current.budget, legacy.budget);
+    assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+        ...legacy,
+        oracleRepairFeedbackVersion: 1
+    }), /digest/u);
+    const { oracleRepairFeedbackVersion: _feedbackVersion, ...stripped } = current;
+    assert.throws(() => assertRepositoryFoundryGenerationPlanV1(stripped), /digest/u);
+    assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+        ...current,
+        oracleRepairFeedbackVersion: 2
+    }));
+    await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+        repositoryRoot: "/does-not-exist-feedback-regression",
+        referenceCommit: "reference",
+        caseId: "case",
+        oracleRepairFeedbackVersion: 2,
+        modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" })
+    })), /repository foundry generation request failed validation/u);
+});

package/dist/test/repository-overlay-isolation.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-overlay-isolation.test.js

@@ -0,0 +1,85 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { replayRepositoryCommandsV1 } from "../adapters/repository-command-runner.js";
+test("all host overlay writers reject external leaf and ancestor links without modifying sentinels", async () => {
+    const root = await mkdtemp(join(tmpdir(), "routekit-overlay-links-"));
+    const outside = await mkdtemp(join(tmpdir(), "routekit-overlay-sentinel-"));
+    const sentinel = join(outside, "sentinel.txt");
+    const git = async (...args) => (await promisify(execFile)("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
+    const commit = async (message) => {
+        await git("add", "-A");
+        await git("commit", "-qm", message);
+        return git("rev-parse", "HEAD");
+    };
+    try {
+        await writeFile(sentinel, "EXTERNAL SENTINEL MUST REMAIN\n");
+        await git("init", "-q");
+        await git("config", "user.name", "Foundry Test");
+        await git("config", "user.email", "foundry@example.test");
+        await writeFile(join(root, "grade.test.cjs"), "const { test } = require('node:test'); test('ordinary grade', () => {});\n");
+        await mkdir(join(root, "src"));
+        await writeFile(join(root, "src/safe.txt"), "original\n");
+        await symlink(sentinel, join(root, "leaf.txt"));
+        await symlink(outside, join(root, "ancestor"));
+        await symlink("src", join(root, "internal-ancestor"));
+        const initial = await commit("initial links");
+        await rm(join(root, "leaf.txt"));
+        await rm(join(root, "ancestor"));
+        await writeFile(join(root, "leaf.txt"), "reviewed reference leaf\n");
+        await mkdir(join(root, "ancestor"));
+        await writeFile(join(root, "ancestor/sentinel.txt"), "reviewed reference descendant\n");
+        const reference = await commit("reference regular files");
+        const common = {
+            repositoryRoot: root,
+            commit: initial,
+            repetitions: 1,
+            recipes: [
+                {
+                    id: "grade",
+                    kind: "test",
+                    cwd: ".",
+                    executable: "node",
+                    args: ["--test", "grade.test.cjs"],
+                    timeoutMs: 10_000
+                }
+            ]
+        };
+        for (const path of ["leaf.txt", "ancestor/sentinel.txt"]) {
+            for (const overrides of [
+                { overlays: [{ path, sourceCommit: reference }] },
+                { contentOverlays: [{ path, content: "ATTEMPTED CONTENT WRITE" }] },
+                { fileOverrides: [{ path, content: "ATTEMPTED FILE WRITE" }] },
+                { fileOverrides: [{ path, delete: true }] }
+            ]) {
+                await assert.rejects(Effect.runPromise(replayRepositoryCommandsV1({ ...common, ...overrides })), /could not materialize/u);
+                assert.equal(await readFile(sentinel, "utf8"), "EXTERNAL SENTINEL MUST REMAIN\n");
+            }
+        }
+        await assert.rejects(Effect.runPromise(replayRepositoryCommandsV1({
+            ...common,
+            fileOverrides: [{ path: "internal-ancestor/safe.txt", content: "linked internal write" }]
+        })), /could not materialize/u);
+        assert.equal(await readFile(join(root, "src/safe.txt"), "utf8"), "original\n");
+        const success = await Effect.runPromise(replayRepositoryCommandsV1({
+            ...common,
+            fileOverrides: [
+                { path: "src/safe.txt", content: "ordinary replacement\n" },
+                { path: "new/nested/file.txt", content: "ordinary new file\n" },
+                { path: "missing.txt", delete: true }
+            ]
+        }));
+        assert.equal(success[0]?.gradeExecuted, true);
+        assert.equal(success[0]?.exitCode, 0);
+        assert.equal(await readFile(sentinel, "utf8"), "EXTERNAL SENTINEL MUST REMAIN\n");
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+        await rm(outside, { recursive: true, force: true });
+    }
+});

package/dist/test/repository-preparation-cache.test.d.ts

@@ -0,0 +1 @@
+export {};