@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/eval-capability-policy.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/eval-capability-policy.test.js

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { evalCapabilityScientificAttemptCostV1 } from "../eval-capability-policy.js";
+const previous = {
+    version: 1,
+    acceptedEvidenceDigest: "a".repeat(64),
+    completedGateDigest: "b".repeat(64),
+    environmentDigest: "c".repeat(64),
+    continuationProgressDigest: "d".repeat(64)
+};
+const continuation = {
+    inputRevision: "candidate",
+    outputRevision: "candidate",
+    previousFingerprint: previous,
+    resultingFingerprint: { ...previous, continuationProgressDigest: "e".repeat(64) },
+    gatesCompleted: [],
+    gatesInvalidated: [],
+    sameFingerprintCount: 0
+};
+test("productive continuation consumes execution budgets but not another scientific attempt", () => {
+    assert.equal(evalCapabilityScientificAttemptCostV1(continuation), 0);
+    assert.equal(evalCapabilityScientificAttemptCostV1({
+        ...continuation,
+        previousFingerprint: { ...previous, continuationProgressDigest: null }
+    }), 0, "the first durable partial work is also a continuation");
+    assert.equal(evalCapabilityScientificAttemptCostV1({
+        ...continuation,
+        resultingFingerprint: {
+            ...continuation.resultingFingerprint,
+            acceptedEvidenceDigest: "f".repeat(64)
+        }
+    }), 0, "an explicitly checkpointed slice may advance partial scientific evidence");
+});
+test("scientific decisions, resets, legacy and unchanged work still consume phase attempts", () => {
+    const changes = [
+        { previousFingerprint: null },
+        { inputRevision: null },
+        { outputRevision: "other-candidate" },
+        { sameFingerprintCount: 1 },
+        { gatesCompleted: ["oracle_authored"] },
+        { gatesInvalidated: ["oracle_authored"] },
+        { resultingFingerprint: previous },
+        { resultingFingerprint: { ...previous, continuationProgressDigest: null } },
+        { resultingFingerprint: { ...continuation.resultingFingerprint, completedGateDigest: "f".repeat(64) } },
+        { resultingFingerprint: { ...continuation.resultingFingerprint, environmentDigest: "f".repeat(64) } }
+    ];
+    for (const change of changes) {
+        assert.equal(evalCapabilityScientificAttemptCostV1({ ...continuation, ...change }), 1);
+    }
+});

package/dist/test/eval-event-log.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/eval-event-log.test.js

@@ -0,0 +1,125 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { Schema } from "effect";
+import { EvalNamedEventLog, assertEvalRunCanFail, assertEvalRunCanComplete, presentEvalEventLog, resumeEvalCaseCursors } from "../eval-event-log.js";
+const at = "2026-08-26T00:00:00.000Z";
+const call = (name, caseId, sequence, dimensionId = "dimension") => ({
+    name,
+    at,
+    runId: "run",
+    dimensionId,
+    caseId,
+    sequence
+});
+test("N=4 candidate-only activity stays per-case and fails before completion", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 80 },
+        ...Array.from({ length: 40 }, (_, index) => call("candidate#", `case-${String((index % 4) + 1)}`, Math.floor(index / 4) + 1))
+    ];
+    const presentation = presentEvalEventLog(events);
+    assert.equal(presentation.failed, true);
+    assert.equal(presentation.lines.length, 4);
+    assert.ok(presentation.lines.every((line) => /fail candidate#1 missing judge#1/u.test(line)));
+    assert.doesNotMatch(presentation.lines.join("\n"), /40\/80|candidate#40/u);
+    assert.throws(() => assertEvalRunCanComplete(events), /cannot complete after candidate#1/u);
+});
+test("resume cursor stops at the first unmatched candidate and does not resume past the hole", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 8 },
+        call("candidate#", "case-a", 1),
+        call("judge#", "case-a", 1),
+        call("candidate#", "case-a", 2),
+        call("candidate#", "case-a", 3),
+        call("judge#", "case-a", 3)
+    ];
+    assert.deepEqual(resumeEvalCaseCursors(events), [
+        {
+            dimensionId: "dimension",
+            caseId: "case-a",
+            completePairs: 1,
+            lastEvent: call("candidate#", "case-a", 2),
+            hole: call("candidate#", "case-a", 2)
+        }
+    ]);
+});
+test("zero planned calls is a failed pulse, not progress", () => {
+    const events = [{ name: "plan", at, runId: "run", plannedCalls: 0 }];
+    assert.deepEqual(presentEvalEventLog(events), {
+        failed: true,
+        lines: ["run fail no planned calls"]
+    });
+    assert.throws(() => assertEvalRunCanComplete(events), /at least one call/u);
+});
+test("CI JSON round-trips the exact same named event log", () => {
+    const events = [
+        { name: "basis", at, operationId: "author-basis" },
+        { name: "dimensions", at, operationId: "author-dimensions" },
+        { name: "activation", at, operationId: "author-activation" },
+        { name: "plan", at, runId: "run", plannedCalls: 2 },
+        call("candidate#", "case-a", 1),
+        call("judge#", "case-a", 1)
+    ];
+    const decoded = Schema.decodeUnknownSync(EvalNamedEventLog)(JSON.parse(JSON.stringify(events)));
+    assert.deepEqual(decoded, events);
+    assert.doesNotThrow(() => assertEvalRunCanComplete(decoded));
+    assert.deepEqual([...new Set(decoded.map((event) => event.name))], ["basis", "dimensions", "activation", "plan", "candidate#", "judge#"]);
+});
+test("authoring heartbeat is the last named artifact event", () => {
+    const events = [
+        { name: "basis", at, operationId: "author-basis" },
+        { name: "dimensions", at, operationId: "author-dimensions" }
+    ];
+    assert.deepEqual(presentEvalEventLog(events, { failHoles: false }), {
+        failed: false,
+        lines: ["authoring dimensions"]
+    });
+});
+test("live presentation keeps a candidate hole in flight and failure presentation names it", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 2 },
+        call("candidate#", "case-a", 1)
+    ];
+    const live = presentEvalEventLog(events, { failHoles: false });
+    assert.deepEqual(live, {
+        failed: false,
+        lines: ["dimension/case-a candidate#1"]
+    });
+    assert.doesNotMatch(live.lines.join("\n"), /missing judge#/u);
+    assert.deepEqual(presentEvalEventLog(events, { failHoles: true }), {
+        failed: true,
+        lines: ["dimension/case-a fail candidate#1 missing judge#1"]
+    });
+});
+test("a terminal failure-detail makes plan-only pre-execution failure durable and bounded", () => {
+    const events = Schema.decodeUnknownSync(EvalNamedEventLog)([
+        { name: "plan", at, runId: "run", plannedCalls: 275 },
+        {
+            name: "failure-detail",
+            at,
+            runId: "run",
+            phase: "target",
+            reason: "evalSession.open rejected while the data gateway was unavailable",
+            retryable: true,
+            accounting: {
+                expectedCalls: 275,
+                observedCalls: 0,
+                knownInputTokens: 0,
+                knownOutputTokens: 0,
+                unknownTokenMeasurements: 0,
+                knownPricedSubtotalUsd: 0,
+                unpricedCalls: 0,
+                subscriptionUnitCalls: 0
+            },
+            cleanup: { sessionOpened: false, sessionClosed: false }
+        }
+    ]);
+    assert.doesNotThrow(() => assertEvalRunCanFail(events));
+    assert.throws(() => assertEvalRunCanComplete(events), /terminal failure/u);
+    assert.deepEqual(presentEvalEventLog(events), {
+        failed: true,
+        lines: ["run fail target: evalSession.open rejected while the data gateway was unavailable"]
+    });
+});
+test("failed eval runs cannot terminate with only a non-terminal plan event", () => {
+    assert.throws(() => assertEvalRunCanFail([{ name: "plan", at, runId: "run", plannedCalls: 275 }]), /terminal failure-detail/u);
+});

package/dist/test/evaluation-evidence-freshness.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/evaluation-evidence-freshness.test.js

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect } from "effect";
+import { assertCurrentEvaluationEvidence } from "../adapters/evaluation-evidence-freshness.js";
+import { compileEvaluationEvidence, selectEvaluationEvidenceBlocks } from "../evaluation-evidence.js";
+import { EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES } from "../evaluation-authoring-policy.js";
+test("freshness revalidates sparse compiler blocks against the complete source", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-evidence-freshness-"));
+    try {
+        const relative = "provider.ts";
+        const content = [
+            "export const alpha = true;\n",
+            "\n",
+            "export const executeProvider = true;\n",
+            "\n",
+            "export const omega = true;\n"
+        ].join("");
+        await writeFile(path.join(root, relative), content);
+        const compiled = compileEvaluationEvidence({
+            sources: [{ path: relative, content }],
+            maximumBlockBytes: EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES
+        });
+        const sparse = selectEvaluationEvidenceBlocks({
+            sources: compiled,
+            evidenceIds: [compiled[0].blocks[0].id]
+        });
+        await Effect.runPromise(assertCurrentEvaluationEvidence({
+            repositoryRoot: root,
+            evidenceSources: sparse
+        }).pipe(Effect.provide(NodeServicesLayer)));
+        await writeFile(path.join(root, relative), content.replace("executeProvider", "executeChangedProvider"));
+        await assert.rejects(Effect.runPromise(assertCurrentEvaluationEvidence({
+            repositoryRoot: root,
+            evidenceSources: sparse
+        }).pipe(Effect.provide(NodeServicesLayer))), /source changed after authoring/u);
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});

package/dist/test/evaluation-evidence.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/evaluation-evidence.test.js

@@ -0,0 +1,230 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { assertEvaluationEvidenceSources, compileEvaluationEvidence, mergeEvaluationEvidenceSources, renderEvaluationEvidenceContext, resolveEvaluationEvidence, selectEvaluationEvidenceBlocks } from "../evaluation-evidence.js";
+test("evidence compilation is deterministic and independent of source input order", () => {
+    const sources = [
+        { path: "src/b.ts", content: "bravo\n" },
+        { path: "src/a.ts", content: "alpha\n" }
+    ];
+    assert.deepEqual(compileEvaluationEvidence({ sources, maximumBlockBytes: 2_048 }), compileEvaluationEvidence({
+        sources: [...sources].reverse(),
+        maximumBlockBytes: 2_048
+    }));
+    assert.deepEqual(compileEvaluationEvidence({ sources, maximumBlockBytes: 2_048 }).map((source) => source.path), ["src/a.ts", "src/b.ts"]);
+});
+test("sparse evidence selection preserves whole-source identity and compiler locations", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [
+            {
+                path: "src/provider.ts",
+                content: [
+                    "export const alpha = true;\n",
+                    "\n",
+                    "export const executeProvider = true;\n",
+                    "\n",
+                    "export const omega = true;\n"
+                ].join("")
+            }
+        ],
+        maximumBlockBytes: 45
+    });
+    const source = compiled[0];
+    assert.ok(source.blocks.length > 1);
+    const selectedBlock = source.blocks[1];
+    const selected = selectEvaluationEvidenceBlocks({
+        sources: compiled,
+        evidenceIds: [selectedBlock.id]
+    });
+    assert.equal(selected[0].sourceDigest, source.sourceDigest);
+    assert.equal(selected[0].sourceBytes, source.sourceBytes);
+    assert.equal(selected[0].sourceBlockCount, source.sourceBlockCount);
+    assert.deepEqual(selected[0].blocks, [selectedBlock]);
+    assertEvaluationEvidenceSources(selected);
+});
+test("sparse evidence merging unions compatible blocks and rejects source drift", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [
+            {
+                path: "src/provider.ts",
+                content: "alpha\n\nprovider backend\n\nresponse decoder\n"
+            }
+        ],
+        maximumBlockBytes: 18
+    });
+    const first = selectEvaluationEvidenceBlocks({
+        sources: compiled,
+        evidenceIds: [compiled[0].blocks[0].id]
+    });
+    const last = selectEvaluationEvidenceBlocks({
+        sources: compiled,
+        evidenceIds: [compiled[0].blocks.at(-1).id]
+    });
+    const merged = mergeEvaluationEvidenceSources([first, last]);
+    assert.deepEqual(merged[0].blocks.map((block) => block.id), [compiled[0].blocks[0].id, compiled[0].blocks.at(-1).id]);
+    assert.throws(() => mergeEvaluationEvidenceSources([
+        first,
+        [
+            {
+                ...last[0],
+                sourceDigest: "0".repeat(64)
+            }
+        ]
+    ]), /identity does not match|changed between authoring packets/u);
+});
+test("evidence identity is bound to canonical path and exact UTF-8 content", () => {
+    const original = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "const label = \"café 🚀\";\n" }],
+        maximumBlockBytes: 2_048
+    })[0];
+    const changedContent = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "const label = \"cafe 🚀\";\n" }],
+        maximumBlockBytes: 2_048
+    })[0];
+    const changedPath = compileEvaluationEvidence({
+        sources: [{ path: "src/renamed.ts", content: "const label = \"café 🚀\";\n" }],
+        maximumBlockBytes: 2_048
+    })[0];
+    assert.notEqual(original.sourceDigest, changedContent.sourceDigest);
+    assert.notEqual(original.blocks[0].id, changedContent.blocks[0].id);
+    assert.notEqual(original.blocks[0].id, changedPath.blocks[0].id);
+    assert.equal(original.blocks[0].text, "const label = \"café 🚀\";\n");
+});
+test("repeated exact text in one source receives distinct compiler-owned identities", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [{ path: "src/repeated.txt", content: "same\nsame\n" }],
+        maximumBlockBytes: 5
+    })[0];
+    assert.equal(compiled.blocks.length, 2);
+    assert.equal(compiled.blocks[0].text, compiled.blocks[1].text);
+    assert.notEqual(compiled.blocks[0].id, compiled.blocks[1].id);
+    assert.deepEqual(resolveEvaluationEvidence({
+        sources: [compiled],
+        evidenceIds: compiled.blocks.map((block) => block.id),
+        maximumContextBytes: 10
+    }).map((block) => block.text), ["same\n", "same\n"]);
+});
+test("evidence blocks prefer blank-line boundaries and obey a hard UTF-8 byte bound", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [
+            {
+                path: "src/example.ts",
+                content: "first line\n\nsecond line\nthird line\n"
+            }
+        ],
+        maximumBlockBytes: 24
+    })[0];
+    assert.equal(compiled.blocks.length, 2);
+    assert.equal(compiled.blocks[0].text, "first line\n\n");
+    assert.equal(compiled.blocks[0].startLine, 1);
+    assert.equal(compiled.blocks[0].endLine, 2);
+    assert.equal(compiled.blocks[1].text, "second line\nthird line\n");
+    assert.ok(compiled.blocks.every((block) => Buffer.byteLength(block.text, "utf8") <= 24));
+});
+test("an oversized line is split without corrupting Unicode or inventing line ranges", () => {
+    const text = "🚀".repeat(9);
+    const compiled = compileEvaluationEvidence({
+        sources: [{ path: "data/emoji.txt", content: text }],
+        maximumBlockBytes: 12
+    })[0];
+    assert.equal(compiled.blocks.map((block) => block.text).join(""), text);
+    assert.deepEqual(compiled.blocks.map((block) => [block.startLine, block.endLine]), [
+        [1, 1],
+        [1, 1],
+        [1, 1]
+    ]);
+    assert.ok(compiled.blocks.every((block) => Buffer.byteLength(block.text, "utf8") <= 12));
+});
+test("evidence resolution rejects unknown, duplicate, and oversized selections", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "one\n\ntwo\n" }],
+        maximumBlockBytes: 6
+    });
+    const ids = compiled[0].blocks.map((block) => block.id);
+    assert.equal(ids.length, 2);
+    assert.throws(() => resolveEvaluationEvidence({
+        sources: compiled,
+        evidenceIds: ["ev_missing"],
+        maximumContextBytes: 12
+    }), /unknown evidence id/u);
+    assert.throws(() => resolveEvaluationEvidence({
+        sources: compiled,
+        evidenceIds: [ids[0], ids[0]],
+        maximumContextBytes: 12
+    }), /duplicate evidence id/u);
+    assert.throws(() => resolveEvaluationEvidence({
+        sources: compiled,
+        evidenceIds: ids,
+        maximumContextBytes: 4
+    }), /exceeds the 4 byte context bound/u);
+});
+test("evidence resolution rejects tampered content, identity, and line metadata", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "one\ntwo\n" }],
+        maximumBlockBytes: 4
+    });
+    const source = compiled[0];
+    const first = source.blocks[0];
+    for (const tampered of [
+        [{ ...source, blocks: [{ ...first, text: "ONE\n" }, ...source.blocks.slice(1)] }],
+        [{ ...source, blocks: [{ ...first, id: "ev_tampered" }, ...source.blocks.slice(1)] }],
+        [{ ...source, blocks: [{ ...first, startLine: 2 }, ...source.blocks.slice(1)] }]
+    ]) {
+        assert.throws(() => assertEvaluationEvidenceSources(tampered), /compiled evidence/u);
+    }
+});
+test("evidence packet merging deduplicates identical sources and rejects drift", () => {
+    const first = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "one\n" }],
+        maximumBlockBytes: 8
+    });
+    const same = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "one\n" }],
+        maximumBlockBytes: 8
+    });
+    const changed = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "two\n" }],
+        maximumBlockBytes: 8
+    });
+    assert.deepEqual(mergeEvaluationEvidenceSources([first, same]), first);
+    assert.throws(() => mergeEvaluationEvidenceSources([first, changed]), /changed between authoring packets/u);
+});
+test("candidate context is mechanically rendered from selected evidence IDs", () => {
+    const compiled = compileEvaluationEvidence({
+        sources: [{ path: "src/example.ts", content: "one\n\ntwo\n" }],
+        maximumBlockBytes: 6
+    });
+    const selected = resolveEvaluationEvidence({
+        sources: compiled,
+        evidenceIds: compiled[0].blocks.map((block) => block.id),
+        maximumContextBytes: 12
+    });
+    assert.equal(renderEvaluationEvidenceContext(selected), selected
+        .map((block) => `Evidence ${block.id} — ${block.path} (lines ${String(block.startLine)}-${String(block.endLine)})\n-----\n${block.text}\n-----`)
+        .join("\n\n"));
+});
+test("evidence compilation rejects duplicate and non-canonical source paths", () => {
+    assert.throws(() => compileEvaluationEvidence({
+        sources: [
+            { path: "src/example.ts", content: "one" },
+            { path: "src/example.ts", content: "two" }
+        ],
+        maximumBlockBytes: 2_048
+    }), /duplicate evidence source path/u);
+    for (const path of [
+        "/absolute.ts",
+        "../escape.ts",
+        "./relative.ts",
+        "src//file.ts",
+        "src/./file.ts",
+        "src\\file.ts"
+    ]) {
+        assert.throws(() => compileEvaluationEvidence({
+            sources: [{ path, content: "one" }],
+            maximumBlockBytes: 2_048
+        }), /not canonical/u);
+    }
+    assert.throws(() => compileEvaluationEvidence({
+        sources: [{ path: "src/empty.ts", content: "" }],
+        maximumBlockBytes: 2_048
+    }), /evidence source is empty/u);
+});

package/dist/test/evaluation-grader-calibration.test.d.ts

@@ -0,0 +1 @@
+export {};