@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-fixture-validation.test.js

@@ -0,0 +1,362 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { RepositoryCommandExecutionPort } from "../adapters/repository-command-runner.js";
+import { makeCaseCheckpointStoreV1 } from "../case-checkpoint-store.js";
+import { RepositoryFoundryError } from "../errors.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { repositoryFixturePreflightFeedbackV1, validateRepositoryFixturesV1 } from "../services/fixture-validation/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+const execute = promisify(execFile);
+const modelPlan = repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" });
+const goodCode = `import assert from "node:assert/strict";
+import test from "node:test";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { value } from "./subject.mjs";
+test("frozen public behavior", () => {
+  assert.equal(existsSync("candidate-ran.txt"), false);
+  writeFileSync("candidate-ran.txt", "fresh candidate only");
+  assert.equal(readFileSync("control.json", "utf8"), "reference");
+  assert.equal(value, 1);
+});
+`;
+const brokenCode = `${goodCode}\nconst syntaxError = ;\n`;
+async function fixture(t) {
+    const root = await realpath(await mkdtemp(join(tmpdir(), "fixture-validation-test-")));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const git = (...args) => execute("git", ["-C", root, ...args]);
+    await git("init", "-q");
+    await writeFile(join(root, "case.test.mjs"), goodCode);
+    const commit = async (value, control) => {
+        await writeFile(join(root, "subject.mjs"), `export const value = ${value};\n`);
+        await writeFile(join(root, "control.json"), control);
+        await git("add", ".");
+        await git("-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false", "-c", "user.name=Fixture Validation", "-c", "user.email=fixture@example.test", "commit", "-qm", "fixture");
+        return (await git("rev-parse", "HEAD")).stdout.trim();
+    };
+    const initialCommit = await commit(0, "initial");
+    const referenceCommit = await commit(1, "reference");
+    // Ambient dirty source must not influence the pinned reference replay.
+    await writeFile(join(root, "control.json"), "untrusted working tree");
+    const grade = {
+        id: "grade",
+        kind: "test",
+        cwd: ".",
+        executable: "node",
+        args: ["--test", "case.test.mjs"],
+        timeoutMs: 5000
+    };
+    const observation = (id, subjectId, outcome) => ({
+        id,
+        kind: "test-command",
+        subjectId,
+        outcome,
+        detail: "Synthetic qualified seed evidence."
+    });
+    const seed = {
+        version: 1,
+        id: "synthetic-seed",
+        status: "qualified",
+        source: { kind: "historical-fix", changeEpisodeId: "synthetic-change" },
+        summary: "Synthetic public behavior.",
+        capabilityEvidence: [{ kind: "test", id: "case", path: "case.test.mjs" }],
+        initialState: { commit: initialCommit },
+        referenceCommit,
+        targetBehavior: [{ id: "behavior", description: "Return one.", critical: true, evidence: [] }],
+        baselineObservations: [observation("baseline", "baseline", "pass")],
+        preChangeObservations: [observation("pre", "grade", "fail")],
+        postChangeObservations: [observation("post", "grade", "pass")],
+        candidateTestIds: ["case"],
+        environment: {
+            protectedControlPaths: ["control.json"],
+            trustedPreparationRecipes: [],
+            candidateValidationRecipes: [
+                {
+                    id: "syntax",
+                    kind: "check",
+                    cwd: ".",
+                    executable: "node",
+                    args: ["--check", "case.test.mjs"],
+                    timeoutMs: 5000
+                }
+            ],
+            baselineGradeRecipes: [{ ...grade, id: "baseline" }],
+            gradeRecipes: [grade]
+        },
+        confidence: "high",
+        risks: [],
+        rejectionReasons: []
+    };
+    const suite = {
+        version: 1,
+        caseId: "synthetic-case",
+        fixtures: ["good", "broken"].map((id) => ({
+            id,
+            description: `Frozen ${id} fixture.`,
+            expectedBehavior: "Return one.",
+            source: "generated",
+            testPath: "case.test.mjs",
+            kind: "metamorphic",
+            expectationMode: "preserved"
+        })),
+        overlays: [
+            { path: "case.test.mjs", content: goodCode, fixtureIds: ["good"] },
+            { path: "case.test.mjs", content: brokenCode, fixtureIds: ["broken"] }
+        ]
+    };
+    return { root, seed, suite };
+}
+function runner(data, options = {}) {
+    const calls = [];
+    const artifacts = [];
+    const events = [];
+    const repairAccounting = { attempts: 0, validatedCalls: 0 };
+    const effect = validateRepositoryFixturesV1({
+        repositoryRoot: data.root,
+        operationId: "fixture-test",
+        seed: options.seed ?? data.seed,
+        suite: options.suite ?? data.suite,
+        allowedTestPaths: new Set(["case.test.mjs"]),
+        context: {
+            visible: { title: "Frozen visible task" },
+            pinnedManifest: { apiVersion: "synthetic" }
+        },
+        modelPlan,
+        maximumRepairAttempts: options.maximumRepairAttempts ?? 1,
+        checkpointStore: options.checkpointStore,
+        onRepairAttempt: () => Effect.sync(() => {
+            repairAccounting.attempts += 1;
+        }),
+        onRepairCall: () => Effect.sync(() => {
+            repairAccounting.validatedCalls += 1;
+        })
+    }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive), Effect.provideService(EvalAuthoringTransport, {
+        complete: (request) => Effect.sync(() => {
+            calls.push(request);
+            return JSON.stringify(options.reply?.(request) ?? {
+                overlays: [{ path: "case.test.mjs", content: goodCode, fixtureIds: ["broken"] }]
+            });
+        })
+    }), Effect.provide(RepositoryFoundryProgressLive((event) => Effect.sync(() => {
+        events.push(event);
+    }), (artifact) => Effect.sync(() => {
+        artifacts.push(artifact);
+    }))));
+    return { effect, calls, artifacts, events, repairAccounting };
+}
+test("fixture preflight repairs only failed code, replays the pinned reference, and retains private evidence without counting host model calls", async (t) => {
+    const data = await fixture(t);
+    const original = JSON.stringify(data.suite);
+    const run = runner(data);
+    const result = await Effect.runPromise(run.effect);
+    assert.equal(result.modelCalls.length, 1);
+    assert.equal(run.calls.length, 1);
+    assert.equal(run.calls[0].foundryRole, "hill-climb-planner");
+    assert.equal(run.calls[0].operationId, "fixture-test:fixture-repair:1");
+    const packet = JSON.parse(run.calls[0].input);
+    assert.deepEqual(packet.failedFixtures.map((entry) => entry.fixture.id), ["broken"]);
+    assert.equal(packet.failedFixtures[0].evidence.outcome, "unknown");
+    assert.equal(packet.failedFixtures[0].diagnostics[0].stage, "candidate-validation");
+    assert.match(packet.failedFixtures[0].diagnostics[0].stderr, /SyntaxError/u);
+    assert.equal(packet.context.pinnedManifest.apiVersion, "synthetic");
+    assert.deepEqual(result.suite.fixtures, data.suite.fixtures);
+    assert.deepEqual(result.suite.overlays[0], data.suite.overlays[0]);
+    assert.equal(result.suite.overlays[1].content, goodCode);
+    assert.equal(JSON.stringify(data.suite), original);
+    const host = run.artifacts.filter((artifact) => artifact.role === "fixture-validator");
+    assert.equal(host.length, 3, "the already-passing overlay is not replayed or rewritten by repair");
+    for (const artifact of host) {
+        const request = artifact.request.input;
+        assert.equal(request.replayInput.commit, data.seed.referenceCommit);
+        assert.equal(request.replayInput.contentOverlays.length, 1);
+        assert.equal(request.replayInput.repetitions, 1);
+        assert.equal(artifact.model, "local-replay");
+        const value = artifact.value;
+        assert.equal(value.admissionEvidence, false);
+        assert.ok(value.results.length > 0);
+    }
+    assert.equal(run.events.filter((event) => event.kind === "model-call" && event.phase === "complete").length, 1);
+    assert.ok(run.events
+        .filter((event) => event.role === "fixture-validator")
+        .every((event) => event.kind === "heartbeat"));
+    assert.ok(run.events.every((event) => !JSON.stringify(event).includes("SyntaxError")));
+    assert.equal(await readFile(join(data.root, "control.json"), "utf8"), "untrusted working tree");
+});
+test("passing reference fixtures require no repair or model calls", async (t) => {
+    const data = await fixture(t);
+    const suite = {
+        ...data.suite,
+        overlays: data.suite.overlays.map((overlay) => ({ ...overlay, content: goodCode }))
+    };
+    const run = runner(data, { suite, maximumRepairAttempts: 0 });
+    const result = await Effect.runPromise(run.effect);
+    assert.deepEqual(result.suite, suite);
+    assert.deepEqual(result.modelCalls, []);
+    assert.equal(run.calls.length, 0);
+    assert.equal(run.events.filter((event) => event.kind === "model-call").length, 0);
+});
+test("reference preflight does not overlap independent fixture command trees", async (t) => {
+    const data = await fixture(t);
+    const suite = {
+        ...data.suite,
+        overlays: data.suite.overlays.map((overlay) => ({ ...overlay, content: goodCode }))
+    };
+    let active = 0;
+    let peak = 0;
+    let executed = 0;
+    const run = runner(data, { suite, maximumRepairAttempts: 0 });
+    const result = await Effect.runPromise(run.effect.pipe(Effect.provideService(RepositoryCommandExecutionPort, {
+        run: (input) => Effect.tryPromise({
+            try: async () => {
+                active += 1;
+                peak = Math.max(peak, active);
+                executed += 1;
+                try {
+                    // Allow a sibling replay to reach its first command if the
+                    // preflight accidentally becomes parallel again.
+                    await new Promise((resolve) => setTimeout(resolve, 1_000));
+                    const { stdout, stderr } = await execute(input.executable, [...input.args], { cwd: input.cwd, env: input.environment, timeout: input.timeoutMs });
+                    return { exitCode: 0, stdout, stderr, timedOut: false };
+                }
+                finally {
+                    active -= 1;
+                }
+            },
+            catch: (cause) => new RepositoryFoundryError({
+                operation: "run-command",
+                detail: "reference preflight command failed",
+                cause
+            })
+        })
+    })));
+    assert.equal(result.modelCalls.length, 0);
+    assert.equal(executed, 4, "both fixtures must run validation and grading");
+    assert.equal(peak, 1, "one fixture's build and grade must finish before the next starts");
+});
+test("failed reference preflight returns concrete compiler evidence to the same author without copying overlay bodies", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, { maximumRepairAttempts: 0 });
+    const result = await Effect.runPromise(run.effect.pipe(Effect.result));
+    assert.equal(result._tag, "Failure");
+    if (result._tag !== "Failure")
+        return;
+    const feedback = repositoryFixturePreflightFeedbackV1(result.failure);
+    assert.match(feedback, /SyntaxError/);
+    assert.match(feedback, /candidate-validation/);
+    assert.match(feedback, /"gradeExecuted":false/);
+    assert.match(feedback, /case\.test\.mjs/);
+    assert.doesNotMatch(feedback, /\[object Object\]/);
+    assert.ok(!feedback.includes(goodCode));
+    assert.ok(feedback.length < 20_000);
+});
+test("fixture preflight retains completed overlays and invalidates only changed executable inputs", async (t) => {
+    const data = await fixture(t);
+    const checkpointStore = makeCaseCheckpointStoreV1({ caseDirectory: join(data.root, "preflight-work") });
+    const suite = {
+        ...data.suite,
+        overlays: data.suite.overlays.map((overlay) => ({ ...overlay, content: goodCode }))
+    };
+    await Effect.runPromise(runner(data, { suite, checkpointStore, maximumRepairAttempts: 0 }).effect);
+    const restored = runner(data, { suite, checkpointStore, maximumRepairAttempts: 0 });
+    await Effect.runPromise(restored.effect.pipe(Effect.provideService(RepositoryCommandExecutionPort, {
+        run: () => Effect.fail(new RepositoryFoundryError({
+            operation: "run-command", detail: "completed preflight must not execute again"
+        }))
+    })));
+    assert.ok(restored.artifacts.filter((artifact) => artifact.role === "fixture-validator")
+        .every((artifact) => artifact.value.restored));
+    const changed = runner(data, {
+        suite: { ...suite, overlays: suite.overlays.map((overlay, index) => index === 0 ? { ...overlay, content: `${goodCode}\n// distinct executable input\n` } : overlay) },
+        checkpointStore, maximumRepairAttempts: 0
+    });
+    await Effect.runPromise(changed.effect);
+    const flags = changed.artifacts.filter((artifact) => artifact.role === "fixture-validator")
+        .map((artifact) => artifact.value.restored).sort();
+    assert.deepEqual(flags, [false, true]);
+});
+test("failed reference preflight stops at its bounded repair allowance before downstream generation", async (t) => {
+    const data = await fixture(t);
+    for (const maximumRepairAttempts of [0, 2]) {
+        const run = runner(data, {
+            maximumRepairAttempts,
+            reply: () => ({ overlays: [data.suite.overlays[1]] })
+        });
+        await assert.rejects(Effect.runPromise(run.effect), /fixture preflight failed after/u);
+        assert.equal(run.calls.length, maximumRepairAttempts);
+        assert.ok(run.calls.every((call) => call.foundryRole === "hill-climb-planner"));
+        assert.equal(run.artifacts.filter((artifact) => artifact.role === "fixture-validator").length, 2 + maximumRepairAttempts);
+    }
+});
+test("repair cannot remove a failed fixture, alter its path or identity, or rewrite a passing overlay", async (t) => {
+    const data = await fixture(t);
+    for (const overlays of [
+        [],
+        [{ path: "other.test.mjs", content: goodCode, fixtureIds: ["broken"] }],
+        [{ path: "case.test.mjs", content: goodCode, fixtureIds: ["renamed"] }],
+        [{ path: "case.test.mjs", content: goodCode, fixtureIds: ["good"] }],
+        [{ path: "case.test.mjs", content: goodCode, fixtureIds: ["broken", "good"] }]
+    ]) {
+        const run = runner(data, { reply: () => ({ overlays }) });
+        await assert.rejects(Effect.runPromise(run.effect), /fixture repair changed frozen identity/u);
+        assert.equal(run.calls.length, 1);
+        assert.equal(run.artifacts.filter((artifact) => artifact.role === "fixture-validator").length, 2);
+    }
+});
+test("invalid fixture metadata and paths fail before any replay or model request", async (t) => {
+    const data = await fixture(t);
+    for (const suite of [
+        { ...data.suite, fixtures: [data.suite.fixtures[0], data.suite.fixtures[0]] },
+        { ...data.suite, overlays: [] },
+        {
+            ...data.suite,
+            fixtures: data.suite.fixtures.map((fixture) => ({
+                ...fixture,
+                testPath: "../case.test.mjs"
+            })),
+            overlays: data.suite.overlays.map((overlay) => ({ ...overlay, path: "../case.test.mjs" }))
+        },
+        {
+            ...data.suite,
+            fixtures: data.suite.fixtures.map((fixture) => ({ ...fixture, expectedBehavior: "" }))
+        }
+    ]) {
+        const run = runner(data, { suite });
+        await assert.rejects(Effect.runPromise(run.effect), /fixture preflight input failed validation/u);
+        assert.equal(run.calls.length, 0);
+        assert.equal(run.artifacts.length, 0);
+    }
+});
+test("a malformed model response still consumes its reserved fixture repair attempt", async (t) => {
+    const data = await fixture(t);
+    const run = runner(data, { reply: () => ({ unexpected: true }) });
+    await assert.rejects(Effect.runPromise(run.effect), /violated routekit_repository_fixture_overlay_repair_v1/u);
+    assert.equal(run.calls.length, 1);
+    assert.deepEqual(run.repairAccounting, { attempts: 1, validatedCalls: 0 });
+    assert.equal(run.events.filter((event) => event.phase === "complete").at(-1)?.modelCalls, 1);
+});
+test("runtime errors, failing setup hooks, and empty test discovery remain unknown and block preflight", async (t) => {
+    const data = await fixture(t);
+    for (const code of [
+        'import test from "node:test"; test("contract", () => { throw new TypeError("runtime"); });',
+        'import test,{beforeEach} from "node:test"; import assert from "node:assert/strict"; beforeEach(() => assert.fail("setup")); test("contract", () => {});',
+        "export const noTests = true;"
+    ]) {
+        const run = runner(data, {
+            maximumRepairAttempts: 0,
+            suite: {
+                ...data.suite,
+                overlays: data.suite.overlays.map((overlay) => ({ ...overlay, content: code }))
+            }
+        });
+        await assert.rejects(Effect.runPromise(run.effect), /fixture preflight failed after/u);
+        assert.equal(run.calls.length, 0);
+        assert.ok(run.artifacts.every((artifact) => artifact.value.evidence.outcome === "unknown"));
+    }
+});

package/dist/test/repository-foundry-progress.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-foundry-progress.test.js

@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { Effect, Layer } from "effect";
+import { boundRepositoryFoundryIdleStallMsV1, formatRepositoryFoundryProgressLineV1, makeRepositoryFoundryProgress, REPOSITORY_FOUNDRY_DEFAULT_IDLE_STALL_MS, reportFoundryModelCallV1, reportFoundryStageV1, RepositoryFoundryIdleStallError, RepositoryFoundryProgress, withRepositoryFoundryIdleStall } from "../index.js";
+test("foundry progress emits stage and model-call events", async () => {
+    const events = [];
+    const progress = await Effect.runPromise(makeRepositoryFoundryProgress((event) => Effect.sync(() => {
+        events.push(event);
+    })));
+    await Effect.runPromise(Effect.gen(function* () {
+        yield* reportFoundryStageV1("author-specification");
+        yield* reportFoundryModelCallV1({ role: "repository-analyst", phase: "start" });
+        yield* reportFoundryModelCallV1({
+            role: "repository-analyst",
+            phase: "complete",
+            callId: "call_1"
+        });
+        yield* reportFoundryStageV1("tournament");
+        const snap = yield* progress.snapshot;
+        assert.equal(snap.stage, "tournament");
+        assert.equal(snap.modelCalls, 1);
+    }).pipe(Effect.provideService(RepositoryFoundryProgress, progress)));
+    assert.equal(events.length, 4);
+    assert.equal(events[0]?.kind, "stage");
+    assert.equal(events[0]?.stage, "author-specification");
+    assert.equal(events[1]?.kind, "model-call");
+    assert.equal(events[1]?.phase, "start");
+    assert.equal(events[2]?.phase, "complete");
+    assert.equal(events[2]?.modelCalls, 1);
+    assert.match(formatRepositoryFoundryProgressLineV1(events[2]), /foundry progress kind=model-call stage=author-specification modelCalls=1 role=repository-analyst phase=complete callId=call_1/u);
+});
+test("idle stall fails closed before first model call when progress stays flat", async () => {
+    const progress = await Effect.runPromise(makeRepositoryFoundryProgress());
+    const startedAt = Date.now();
+    const error = await Effect.runPromise(withRepositoryFoundryIdleStall(Effect.gen(function* () {
+        yield* reportFoundryStageV1("execute-start");
+        // Silent post-turbo authoring: no further stage/modelCalls emissions.
+        yield* Effect.sleep(5_000);
+        return "should-not-reach";
+    }), {
+        idleStallMs: 80,
+        wallTimeMs: 10_000
+    }).pipe(Effect.provideService(RepositoryFoundryProgress, progress), Effect.flip));
+    assert.ok(error instanceof RepositoryFoundryIdleStallError);
+    assert.equal(error.stage, "execute-start");
+    assert.equal(error.modelCalls, 0);
+    assert.match(error.message, /stalled/u);
+    assert.match(error.message, /idleStallMs=80/u);
+    assert.ok(Date.now() - startedAt < 2_000);
+});
+test("idle stall fails closed after a hung model-call start with no completion", async () => {
+    const progress = await Effect.runPromise(makeRepositoryFoundryProgress());
+    const startedAt = Date.now();
+    const error = await Effect.runPromise(withRepositoryFoundryIdleStall(Effect.gen(function* () {
+        yield* reportFoundryStageV1("author-specification");
+        yield* reportFoundryModelCallV1({
+            role: "repository-analyst",
+            phase: "start"
+        });
+        // Hung provider call or stalled transport: start emitted, nothing else.
+        yield* Effect.sleep(5_000);
+        return "should-not-reach";
+    }), {
+        idleStallMs: 80,
+        wallTimeMs: 10_000
+    }).pipe(Effect.provideService(RepositoryFoundryProgress, progress), Effect.flip));
+    assert.ok(error instanceof RepositoryFoundryIdleStallError);
+    assert.equal(error.stage, "author-specification");
+    assert.equal(error.modelCalls, 0);
+    assert.match(error.message, /stalled/u);
+    assert.ok(Date.now() - startedAt < 2_000);
+});
+test("normal progress does not false-trigger idle stall", async () => {
+    const progress = await Effect.runPromise(makeRepositoryFoundryProgress());
+    const result = await Effect.runPromise(withRepositoryFoundryIdleStall(Effect.gen(function* () {
+        for (let index = 0; index < 5; index += 1) {
+            yield* Effect.sleep(40);
+            yield* reportFoundryModelCallV1({
+                role: "oracle-designer",
+                phase: index % 2 === 0 ? "start" : "complete"
+            });
+        }
+        return "ok";
+    }), {
+        idleStallMs: 120,
+        wallTimeMs: 10_000
+    }).pipe(Effect.provideService(RepositoryFoundryProgress, progress)));
+    assert.equal(result, "ok");
+    const snap = await Effect.runPromise(progress.snapshot);
+    assert.equal(snap.modelCalls, 2);
+});
+test("idle stall is capped by wall and defaults to twenty minutes", () => {
+    assert.equal(REPOSITORY_FOUNDRY_DEFAULT_IDLE_STALL_MS, 20 * 60_000);
+    assert.equal(boundRepositoryFoundryIdleStallMsV1(undefined, 4 * 60 * 60_000), 20 * 60_000);
+    assert.equal(boundRepositoryFoundryIdleStallMsV1(60_000, 30_000), 30_000);
+});
+test("progress helpers no-op without a progress service", async () => {
+    await Effect.runPromise(Effect.gen(function* () {
+        yield* reportFoundryStageV1("execute-start");
+        yield* reportFoundryModelCallV1({ role: "adversary", phase: "start" });
+    }));
+});
+test("RepositoryFoundryProgressLive layer wires emit", async () => {
+    const events = [];
+    const layer = Layer.effect(RepositoryFoundryProgress, makeRepositoryFoundryProgress((event) => Effect.sync(() => {
+        events.push(event.stage);
+    })));
+    await Effect.runPromise(reportFoundryStageV1("admit").pipe(Effect.provide(layer)));
+    assert.deepEqual(events, ["admit"]);
+});

package/dist/test/repository-foundry-quality.test.d.ts

@@ -0,0 +1 @@
+export {};