@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/case-pipeline-v2.test.js

@@ -0,0 +1,286 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { Effect } from "effect";
+import { PipelinePortErrorV2, pipelineCheckpointDescriptorDigestV2, pipelineScientificIdentityDigestV2 } from "../case-pipeline-protocol-v2.js";
+import { makePortableCheckpointStoreV2, repositoryBehaviorMapDigestV2, runCasePipelineV2 } from "../services/case-pipeline-v2/service.js";
+import { makePortableRepositoryBundleV2 } from "../services/repository-bundle/service.js";
+const sha = "a".repeat(64);
+const repository = {
+    repositoryId: "repo-1",
+    repository: "velum-labs/routekit",
+    commitSha: "b".repeat(40),
+    treeSha: "c".repeat(40)
+};
+const map = {
+    version: 1,
+    repository: {
+        root: "/ignored/runtime/path",
+        requestedRef: repository.commitSha,
+        commit: repository.commitSha,
+        tree: repository.treeSha
+    },
+    packages: [],
+    publicSurfaces: [],
+    protocols: [],
+    tests: [],
+    testBindings: [],
+    commands: [],
+    historyEpisodes: [],
+    workloadSignals: [],
+    blindSpots: []
+};
+const identity = {
+    version: 2,
+    repository,
+    caseId: "case-1",
+    coverageCellId: "cell-1",
+    dimension: {
+        id: "routing",
+        description: "Routing behavior",
+        includes: ["explicit behavior"],
+        excludes: ["unrelated formatting"]
+    },
+    source: {
+        referenceCommitSha: "d".repeat(40),
+        referenceTaskDigest: sha,
+        basisDigest: "e".repeat(64)
+    },
+    modelPlan: [
+        "seed-author",
+        "specification-author",
+        "specification-reviewer",
+        "oracle-author",
+        "control-author",
+        "adversary-author",
+        "oracle-critic",
+        "admission-reviewer"
+    ].map((role) => ({
+        role: role,
+        model: "openai/fake",
+        reasoningEffort: "high",
+        maximumCalls: 100,
+        maximumOutputTokens: 65_536
+    })),
+    policy: {
+        foundryPolicyVersion: "foundry-v2",
+        admissionPolicyVersion: "admission-v1",
+        repositoryModelDigest: repositoryBehaviorMapDigestV2(map),
+        oracleRepetitions: 2,
+        oracleConcurrency: 2,
+        foldLimit: 2
+    },
+    budget: {
+        maximumModelCalls: 100,
+        maximumWallTimeMs: 60_000,
+        maximumArtifactBytes: 16 * 1024 * 1024
+    }
+};
+const portFailure = (port, operation) => new PipelinePortErrorV2({
+    port,
+    operation,
+    detail: `${port} ${operation} failed`,
+    retryable: false
+});
+const makePorts = (stored) => ({
+    repository: {
+        materialize: () => Effect.void,
+        verify: () => Effect.void
+    },
+    model: {
+        complete: () => Effect.fail(portFailure("model", "unexpected-call"))
+    },
+    checkpoints: {
+        list: () => Effect.succeed(stored.map(({ descriptor }) => descriptor)),
+        read: (descriptor) => {
+            const found = stored.find(({ descriptor: candidate }) => candidate.checkpointId === descriptor.checkpointId);
+            return found === undefined
+                ? Effect.fail(portFailure("checkpoint", "read"))
+                : Effect.succeed(found.bytes);
+        },
+        put: (descriptor, bytes) => {
+            stored.push({ descriptor, bytes });
+            return Effect.succeed(descriptor);
+        }
+    },
+    artifacts: {
+        put: (descriptor) => Effect.succeed(descriptor),
+        read: () => Effect.fail(portFailure("artifact", "read"))
+    },
+    execution: {
+        run: () => Effect.fail(portFailure("execution", "run"))
+    },
+    progress: {
+        emit: () => Effect.void
+    },
+    clock: {
+        now: Effect.succeed(new Date("2026-09-11T00:00:00.000Z")),
+        sleep: () => Effect.void
+    }
+});
+test("portable runner appends one accepted terminal head after the durable admit result", async () => {
+    const firstRoot = await mkdtemp(join(tmpdir(), "routekit-foundry-v2-terminal-a-"));
+    const secondRoot = await mkdtemp(join(tmpdir(), "routekit-foundry-v2-terminal-b-"));
+    const stored = [];
+    const admit = {
+        version: 1,
+        stage: "admit",
+        parentDigest: sha,
+        createdAt: "2026-09-11T00:00:00.000Z",
+        verdict: "admitted",
+        reasons: [],
+        benchmarkCase: {},
+        evidence: {
+            validCount: 1,
+            wrongCount: 1,
+            heldOutWrongCount: 1,
+            semanticKills: ["wrong-1"]
+        }
+    };
+    try {
+        const first = await Effect.runPromise(makePortableCheckpointStoreV2({
+            identity,
+            runtimeRoot: firstRoot,
+            attemptGeneration: 1,
+            ports: makePorts(stored)
+        }));
+        await Effect.runPromise(first.store.writeStage("admit", admit));
+        assert.deepEqual(JSON.parse(await readFile(first.store.stagePath("admit"), "utf8")), admit, "the V1 admit result is durable before terminalization");
+        const stageHead = first.descriptors().at(-1);
+        assert.ok(stageHead);
+        assert.equal(stageHead.kind, "stage");
+        const accepted = await Effect.runPromise(first.completeAdmit(admit));
+        const replay = await Effect.runPromise(first.completeAdmit(admit));
+        assert.deepEqual(replay, accepted);
+        assert.equal(stored.length, 2);
+        assert.equal(accepted.kind, "terminal");
+        assert.equal(accepted.stage, "admit");
+        assert.equal(accepted.parentDigest, pipelineCheckpointDescriptorDigestV2(stageHead));
+        assert.deepEqual(first.descriptors().at(-1), accepted);
+        assert.equal(first.descriptors().filter(({ kind }) => kind === "terminal").length, 1);
+        const replacement = await Effect.runPromise(makePortableCheckpointStoreV2({
+            identity,
+            runtimeRoot: secondRoot,
+            attemptGeneration: 2,
+            ports: makePorts(stored)
+        }));
+        const replacementReplay = await Effect.runPromise(replacement.completeAdmit(admit));
+        assert.deepEqual(replacementReplay, accepted);
+        await assert.rejects(Effect.runPromise(replacement.completeAdmit({
+            ...admit,
+            verdict: "rejected",
+            reasons: ["replacement changed the durable result"]
+        })), /terminal checkpoint replay conflicts with the durable admit result/i);
+        assert.equal(stored.length, 2, "replacement resume must not replace or extend the terminal head");
+        assert.deepEqual(replacement.descriptors().at(-1), accepted);
+    }
+    finally {
+        await Promise.all([
+            rm(firstRoot, { recursive: true, force: true }),
+            rm(secondRoot, { recursive: true, force: true })
+        ]);
+    }
+});
+test("portable runner resumes accepted terminal work at a different runtime path", async () => {
+    const firstRoot = await mkdtemp(join(tmpdir(), "routekit-foundry-v2-a-"));
+    const secondRoot = await mkdtemp(join(tmpdir(), "routekit-foundry-v2-b-"));
+    const stored = [];
+    try {
+        const identityDigest = pipelineScientificIdentityDigestV2(identity);
+        const payload = Buffer.from(JSON.stringify({
+            version: 2,
+            action: "write",
+            stage: "admit",
+            value: {
+                version: 1,
+                stage: "admit",
+                parentDigest: sha,
+                createdAt: "2026-09-11T00:00:00.000Z",
+                verdict: "admitted",
+                reasons: [],
+                benchmarkCase: {}
+            }
+        }), "utf8");
+        const payloadDigest = await crypto.subtle.digest("SHA-256", payload);
+        stored.push({
+            descriptor: {
+                version: 2,
+                checkpointId: "checkpoint-1",
+                identityDigest,
+                attemptGeneration: 1,
+                sequence: 1,
+                kind: "terminal",
+                stage: "admit",
+                parentDigest: identityDigest,
+                payloadDigest: Buffer.from(payloadDigest).toString("hex"),
+                payloadBytes: payload.byteLength,
+                createdAt: "2026-09-11T00:00:00.000Z"
+            },
+            bytes: payload
+        });
+        /*
+         * A deliberately incomplete admit payload is rejected by the existing V1
+         * schema and regenerated. The proof here is that restore reaches the V1
+         * implementation through the new port layer without path-identity failure;
+         * the fake model then fails closed instead of silently accepting it.
+         */
+        const first = await Effect.runPromise(runCasePipelineV2({
+            identity,
+            runtimeRoot: firstRoot,
+            attemptGeneration: 2,
+            map,
+            ports: makePorts(stored)
+        }));
+        const second = await Effect.runPromise(runCasePipelineV2({
+            identity,
+            runtimeRoot: secondRoot,
+            attemptGeneration: 3,
+            map,
+            ports: makePorts(stored)
+        }));
+        assert.equal(first.identityDigest, second.identityDigest);
+        assert.notEqual(first.runtimeRoot, second.runtimeRoot);
+        assert.match(first.result.state.reason ?? "", /accepted terminal checkpoint is already the immutable chain head/i);
+        assert.match(second.result.state.reason ?? "", /accepted terminal checkpoint is already the immutable chain head/i);
+        assert.doesNotMatch(first.result.state.reason ?? "", /different pipeline identity/i);
+    }
+    finally {
+        await Promise.all([
+            rm(firstRoot, { recursive: true, force: true }),
+            rm(secondRoot, { recursive: true, force: true })
+        ]);
+    }
+});
+test("portable repository bundle is deterministic and path-safe", () => {
+    const input = {
+        admitted: true,
+        sourceCommitSha: identity.repository.commitSha,
+        artifacts: [
+            {
+                path: ".routekit/evals/b.json",
+                kind: "public-metadata",
+                mediaType: "application/json",
+                content: "{}\n"
+            },
+            {
+                path: ".routekit/evals/a.eval.ts",
+                kind: "eval-definition",
+                mediaType: "text/typescript",
+                content: "export {};\n"
+            }
+        ]
+    };
+    const first = makePortableRepositoryBundleV2(input);
+    const second = makePortableRepositoryBundleV2({
+        ...input,
+        artifacts: [...input.artifacts].reverse()
+    });
+    assert.deepEqual(first, second);
+    assert.deepEqual(first.files.map(({ path }) => path), [".routekit/evals/a.eval.ts", ".routekit/evals/b.json"]);
+    assert.throws(() => makePortableRepositoryBundleV2({
+        ...input,
+        artifacts: [{ ...input.artifacts[0], path: "../secret" }]
+    }));
+});

package/dist/test/case-pipeline.test.d.ts

@@ -0,0 +1 @@
+export {};