@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-review-evidence-v2.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-review-evidence-v2.test.js

@@ -0,0 +1,183 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { decodeRepositoryReviewEvidenceV2, encodeRepositoryReviewEvidenceV1, encodeRepositoryReviewEvidenceV2 } from "../adapters/repository-review-evidence.js";
+const wire = (value) => JSON.stringify(value);
+const bytes = (value) => Buffer.byteLength(wire(value));
+test("compact review encoding preserves arbitrary JSON, source changes, and string delimiters", () => {
+    const source = `${"export const stable = true;\n".repeat(200)}😀 selected\n`;
+    const input = {
+        sources: [{ content: source }, { content: source.replace("selected", "changed") }],
+        values: [null, true, false, 0, undefined, ["row", 0, "literal array"]],
+        diagnostics: [
+            "recipe=retained-behavior; exit=0; timeout=false; assertions=23; durationMs=102; exactly repeated delimiter field;".repeat(2),
+            "recipe=retained-behavior; exit=1; timeout=false; assertions=23; durationMs=103; exactly repeated delimiter field;"
+        ],
+        special: Object.fromEntries([
+            ["__proto__", { kept: true }],
+            ["toJSON", "literal key"],
+            ["encoding", ["string", 0]],
+            ["unicode", ["\ud800".repeat(256), "é😀\r\n"]]
+        ])
+    };
+    const packet = encodeRepositoryReviewEvidenceV2(input, [source]);
+    assert.equal(wire(decodeRepositoryReviewEvidenceV2(packet)), wire(input));
+    assert.equal(wire(decodeRepositoryReviewEvidenceV2(JSON.parse(wire(packet)))), wire(input));
+});
+test("compact review retains full executed populations, including failing and unknown observations", () => {
+    const source = `${"export const unchanged = true;\n".repeat(900)}export const selected = 0;\n`;
+    const observation = (index, phase) => ({
+        solutionId: `independently-authored-${phase}-control-${index % 18}`,
+        clauseId: `required-executable-behavior-clause-${index % 12}`,
+        repetition: (index % 2) + 1,
+        outcome: index % 19 === 0 ? "unknown" : index % 13 === 0 ? "fail" : "pass",
+        detail: [
+            "recipe=command-complete-targeted-tests",
+            "observedRecipe=command-complete-targeted-tests",
+            `exit=${index % 13 === 0 ? 1 : 0}`,
+            "stage=grade",
+            "gradeExecuted=true",
+            "timeout=false",
+            `durationMs=${index + 200}`,
+            `preparationTemplate=${"a".repeat(64)}`,
+            `cloneDurationMs=${index + 1000}`,
+            "evidence=host reporter recorded complete named assertions"
+        ].join("; ")
+    });
+    const input = {
+        sources: [{ path: "src/example.ts", content: source }],
+        solutions: Array.from({ length: 30 }, (_, index) => ({
+            id: `independent-control-${index}`,
+            content: source.replace("selected = 0", `selected = ${index + 1}`)
+        })),
+        development: Array.from({ length: 460 }, (_, index) => observation(index, "development")),
+        heldOut: Array.from({ length: 540 }, (_, index) => observation(index, "held-out")),
+        evidenceInventory: Array.from({ length: 540 }, (_, index) => `review-packet:/heldOut/${index}`)
+    };
+    const old = encodeRepositoryReviewEvidenceV1(input, [source]);
+    const compact = encodeRepositoryReviewEvidenceV2(input, [source]);
+    assert.ok(bytes(old) > 512_000);
+    assert.ok(bytes(compact) < 512_000);
+    assert.ok(bytes(compact) < bytes(old) * 0.8);
+    const decoded = decodeRepositoryReviewEvidenceV2(compact);
+    assert.equal(wire(decoded), wire(input));
+    assert.equal(decoded.heldOut.length, 540);
+    assert.equal(decoded.development.length, 460);
+    assert.deepEqual(decoded.heldOut.filter((entry) => entry.outcome !== "pass"), input.heldOut.filter((entry) => entry.outcome !== "pass"));
+});
+test("compact review rejects changed outcomes, missing columns, invalid references, and digest changes", () => {
+    const packet = encodeRepositoryReviewEvidenceV2({ solutionId: "independent-solution", outcome: "unknown", observations: [true] }, []);
+    const damaged = (edit) => {
+        const copy = JSON.parse(wire(packet));
+        edit(copy);
+        return copy;
+    };
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.evidence[3] = "pass";
+    })), /original evidence digest mismatch/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.evidence.pop();
+    })), /object shape/u);
+    for (const invalid of [-1, 0.5, 99])
+        assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+            copy.evidence[1] = invalid;
+        })), /shape reference is missing/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.shapes[0][1] = copy.shapes[0][0];
+    })), /unique object shape/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.evidence[2] = ["j", "text", 7];
+    })), /only literal string parts/u);
+});
+test("shared review subtrees retain every occurrence and reject cycles and changed shared outcomes", () => {
+    const observed = {
+        id: "complete-frozen-control-population",
+        observations: Array.from({ length: 20 }, (_, index) => ({
+            solutionId: `retained-control-${index}`,
+            outcome: index === 7 ? "unknown" : "pass",
+            repetition: 2,
+            detail: "Retain this complete executed observation in every population."
+        }))
+    };
+    const input = { first: observed, second: observed, nested: [{ retained: observed }] };
+    const packet = encodeRepositoryReviewEvidenceV2(input, []);
+    assert.ok(packet.nodes.length > 0);
+    assert.deepEqual(decodeRepositoryReviewEvidenceV2(packet), input);
+    assert.ok(bytes(packet) < bytes(encodeRepositoryReviewEvidenceV1(input, [])));
+    const damaged = {
+        ...packet,
+        nodes: [["n", 0], ...packet.nodes.slice(1)]
+    };
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged), /acyclic and point backwards/u);
+    const changed = {
+        ...packet,
+        nodes: [["a", "changed outcome"], ...packet.nodes.slice(1)]
+    };
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(changed), /original evidence digest mismatch/u);
+});
+test("separated source edits retain unchanged regions, Unicode, CRLF, and complete control populations", () => {
+    const source = Array.from({ length: 480 }, (_, index) => `export const unchanged_${index} = ${index}; // retained 😀\r\n`).join("");
+    const variants = Array.from({ length: 18 }, (_, index) => source
+        .replace("unchanged_12 = 12", `unchanged_12 = ${1000 + index}`)
+        .replace("unchanged_415 = 415", `unchanged_415 = ${2000 + index}`)
+        .concat(`export const added_${index} = "é";\r\n`));
+    const input = {
+        source,
+        controls: variants.map((content, index) => ({
+            id: `independent-control-${index}`,
+            content,
+            outcome: index === 6 ? "unknown" : index === 8 ? "fail" : "pass"
+        }))
+    };
+    const compact = encodeRepositoryReviewEvidenceV2(input, [source]);
+    const legacy = encodeRepositoryReviewEvidenceV1(input, [source]);
+    assert.ok(compact.texts.some(([, text]) => text[0] === "patch"));
+    assert.ok(bytes(compact) < bytes(legacy) * 0.3);
+    assert.deepEqual(decodeRepositoryReviewEvidenceV2(compact), input);
+    assert.deepEqual(decodeRepositoryReviewEvidenceV2(JSON.parse(wire(compact))), input);
+});
+test("source patches reject overlapping edits, incorrect context, modified replacements, and cycles", () => {
+    const source = Array.from({ length: 300 }, (_, index) => `export const retained_${index} = ${index};\n`).join("");
+    const changed = source
+        .replace("retained_10 = 10", "retained_10 = -1")
+        .replace("retained_270 = 270", "retained_270 = -2");
+    const packet = encodeRepositoryReviewEvidenceV2({ source, changed }, [source]);
+    const index = packet.texts.findIndex(([, entry]) => entry[0] === "patch");
+    assert.ok(index >= 0);
+    const damaged = (edit) => {
+        const copy = JSON.parse(wire(packet));
+        edit(copy);
+        return copy;
+    };
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        const edits = copy.texts[index][1][2];
+        edits[1][0] = edits[0][0];
+    })), /invalid ranges or source context/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.texts[index][1][2][0][3].startLine += 1;
+    })), /invalid ranges or source context/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.texts[index][1][2][0][2] += "invalid";
+    })), /text digest mismatch/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV2(damaged((copy) => {
+        copy.texts[index][1][1] = copy.texts[index][0];
+    })), /bounded and acyclic/u);
+});
+test("source patch round trips preserve arbitrary insertion and deletion positions", () => {
+    let seed = 1729;
+    const random = (maximum) => {
+        seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
+        return seed % maximum;
+    };
+    const source = Array.from({ length: 90 }, (_, index) => `const retained_${index} = "${index}😀";\n${index % 3 === 0 ? "\n" : ""}`).join("");
+    for (let iteration = 0; iteration < 60; iteration += 1) {
+        let changed = source;
+        for (let edit = 0; edit < 6; edit += 1) {
+            const start = random(changed.length + 1);
+            const end = Math.min(changed.length, start + random(50));
+            const replacement = ["", "😀\r\n", `value_${iteration}_${edit}`, "\ud800"][random(4)];
+            changed = changed.slice(0, start) + replacement + changed.slice(end);
+        }
+        const input = { source, changed, duplicate: changed };
+        assert.equal(wire(decodeRepositoryReviewEvidenceV2(encodeRepositoryReviewEvidenceV2(input, [source]))), wire(input));
+    }
+});

package/dist/test/repository-review-evidence.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-review-evidence.test.js

@@ -0,0 +1,124 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { decodeRepositoryReviewEvidenceV1, encodeRepositoryReviewEvidenceV1 } from "../adapters/repository-review-evidence.js";
+const wire = (value) => JSON.stringify(value);
+const bytes = (value) => Buffer.byteLength(wire(value));
+test("review encoding preserves every JSON byte, including Unicode, reserved-looking keys, arrays and full-file edits", () => {
+    const base = `${"export const retained = true;\n".repeat(200)}😀 original\n${"// unchanged suffix\n".repeat(200)}`;
+    const variants = [
+        base.replace("😀 original", "😃 repaired"),
+        `added at start\n${base}`,
+        `${base}\nadded at end`,
+        base.replace("😀 original", "")
+    ];
+    const input = {
+        sources: [{ path: "src/wire.ts", content: base }],
+        controls: variants.map((content, index) => ({
+            id: String(index),
+            fileOverrides: [{ content }]
+        })),
+        observations: variants.map((_, index) => ({
+            index,
+            outcome: "unknown",
+            detail: "Exact evidence stays present."
+        })),
+        arbitrary: Object.fromEntries([
+            ["__proto__", { important: true }],
+            ["$foundryText", ["text", "a".repeat(64)]],
+            ["encoding", "routekit-lossless-review-json-v1"],
+            ["toJSON", "ordinary JSON field"],
+            ["unicode", ["\ud800".repeat(256), "\ud801".repeat(256), "é😀\r\n"]]
+        ]),
+        optional: undefined,
+        arrayWithMissingValue: [undefined, null, false, 0]
+    };
+    const packet = encodeRepositoryReviewEvidenceV1(input, [base]);
+    assert.equal(wire(decodeRepositoryReviewEvidenceV1(packet)), wire(input));
+    assert.equal(wire(decodeRepositoryReviewEvidenceV1(JSON.parse(wire(packet)))), wire(input));
+    assert.ok(Object.values(packet.texts).some((entry) => entry[0] === "splice"));
+    assert.ok(bytes(packet) < bytes(input));
+});
+test("review reconstruction rejects changed text, missing bases and overlapping UTF16 ranges", () => {
+    const base = "😀 original behavior\n".repeat(300);
+    const packet = encodeRepositoryReviewEvidenceV1({ content: base.replace("original", "repaired") }, [base]);
+    const spliceKey = Object.keys(packet.texts).find((key) => packet.texts[key]?.[0] === "splice");
+    assert.ok(spliceKey);
+    const damaged = (edit) => {
+        const copy = JSON.parse(wire(packet));
+        edit(copy.texts);
+        return copy;
+    };
+    assert.throws(() => decodeRepositoryReviewEvidenceV1(damaged((texts) => {
+        const splice = texts[spliceKey];
+        splice[4] = "different source";
+    })), /digest mismatch/u);
+    assert.throws(() => decodeRepositoryReviewEvidenceV1(damaged((texts) => {
+        const splice = texts[spliceKey];
+        delete texts[splice[1]];
+    })), /invalid base/u);
+    for (const invalidPrefix of [-1, 0.5, base.length + 1]) {
+        assert.throws(() => decodeRepositoryReviewEvidenceV1(damaged((texts) => {
+            const splice = texts[spliceKey];
+            splice[2] = invalidPrefix;
+        })), /overlapping ranges/u);
+    }
+    assert.throws(() => decodeRepositoryReviewEvidenceV1(damaged((texts) => {
+        const splice = texts[spliceKey];
+        splice[5] = {
+            startLine: 1,
+            endLine: 1,
+            immediatelyBefore: "misleading source",
+            immediatelyAfter: ""
+        };
+    })), /context does not match/u);
+});
+test("v4-sized development and held-out populations fit unchanged review cap with exact full evidence", () => {
+    // Synthetic ordinary source, sized after the saved v4 population. No private task or oracle data.
+    const source = `${Array.from({ length: 390 }, (_, i) => `export const value${i} = ${i}; // retained public behavior\n`).join("")}export const selected = 0;\n`;
+    const solution = (population, index) => ({
+        id: `${population}-${index}`,
+        kind: index < 6 ? "mutation" : "reward-hacking",
+        family: `${population}-family-${index}`,
+        fileOverrides: [
+            {
+                path: "src/wire.ts",
+                content: source.replace("selected = 0", `selected = ${index + (population === "held-out" ? 30 : 1)}`)
+            }
+        ]
+    });
+    const input = {
+        repositoryEnvironment: { sources: [{ path: "src/wire.ts", content: source }] },
+        visible: { requestText: "Preserve the public protocol behavior." },
+        validSolutions: [solution("valid", 0), solution("valid", 1)],
+        developmentAdversaries: Array.from({ length: 9 }, (_, i) => solution("development", i)),
+        heldOutAdversaries: Array.from({ length: 9 }, (_, i) => solution("held-out", i)),
+        fixtureProposal: {
+            fixtures: Array.from({ length: 10 }, (_, i) => ({
+                id: `fixture-${i}`,
+                expectedBehavior: `Requirement ${i}`
+            })),
+            overlays: Array.from({ length: 10 }, (_, i) => ({
+                path: "test/wire.test.ts",
+                content: `// fixture ${i}\n${"assert.equal(actual, expected);\n".repeat(160)}`
+            }))
+        },
+        evidenceInventory: ["review-packet:/heldOutAdversaries/8", "review-packet:/validSolutions/1"]
+    };
+    assert.ok(bytes(input) > 512_000, `unencoded fixture was only ${bytes(input)} bytes`);
+    const packet = encodeRepositoryReviewEvidenceV1(input, [source]);
+    assert.ok(bytes(packet) < 512_000, `encoded fixture was ${bytes(packet)} bytes`);
+    assert.equal(wire(decodeRepositoryReviewEvidenceV1(packet)), wire(input));
+    const restored = decodeRepositoryReviewEvidenceV1(packet);
+    assert.equal(restored.heldOutAdversaries.length, 9);
+    assert.equal(restored.developmentAdversaries.length, 9);
+    assert.deepEqual(restored.evidenceInventory, input.evidenceInventory);
+});
+test("encoding remains bounded before creating a whole oversized JSON request", () => {
+    for (const input of [new Date(0), { toJSON: () => ({ hiddenTransformation: true }) }])
+        assert.throws(() => encodeRepositoryReviewEvidenceV1(input, []), /plain JSON/u);
+    assert.throws(() => encodeRepositoryReviewEvidenceV1({ source: "x".repeat(16 * 1024 * 1024) }, []), /16MiB encoding bound/u);
+    let nested = "leaf";
+    for (let i = 0; i < 130; i += 1)
+        nested = [nested];
+    assert.throws(() => encodeRepositoryReviewEvidenceV1(nested, []), /depth bound/u);
+});

package/dist/test/repository-seed-exclusions.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-seed-exclusions.test.js

@@ -0,0 +1,96 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { assertRepositoryFoundryGenerationPlanV1, assertRepositoryFoundryReferenceExclusionsV1, repositoryFoundryGenerationPlanDigestV1 } from "../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { planHistoricalRepositoryCaseGenerationV1 } from "../services/foundry/service.js";
+const exec = promisify(execFile);
+const modelPlan = repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" });
+async function fixture(t) {
+    const root = await realpath(await mkdtemp(join(tmpdir(), "seed-exclusion-unit-")));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    await mkdir(join(root, "src"));
+    await mkdir(join(root, "test"));
+    await writeFile(join(root, "package.json"), JSON.stringify({
+        name: "protocol-value-fixture",
+        type: "module",
+        scripts: { test: "node --test test/*.test.js" }
+    }));
+    const git = (...args) => exec("git", ["-C", root, ...args]);
+    await git("init", "-q");
+    const commits = [];
+    for (let value = 0; value <= 2; value += 1) {
+        await writeFile(join(root, "src/index.js"), `export const protocolValue = () => ${value};\n`);
+        await writeFile(join(root, "test/value.test.js"), `import assert from 'node:assert/strict';\nimport { test } from 'node:test';\nimport { protocolValue } from '../src/index.js';\ntest('protocol value', () => assert.equal(protocolValue(), ${value}));\n`);
+        await git("add", ".");
+        await git("-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-qm", value === 0 ? "initial fixture" : `fix: protocol value regression ${value}`);
+        commits.push((await git("rev-parse", "HEAD")).stdout.trim());
+    }
+    return { root, references: commits.slice(1) };
+}
+test("admitted references are removed before the one-candidate qualification budget", async (t) => {
+    const f = await fixture(t);
+    const input = {
+        repositoryRoot: f.root,
+        caseId: "first-selection",
+        requestedDimension: "protocol value",
+        maximumSeedCandidates: 1,
+        modelPlan
+    };
+    const first = await Effect.runPromise(planHistoricalRepositoryCaseGenerationV1(input));
+    assert.ok(f.references.includes(first.referenceCommit));
+    assert.equal(first.excludedReferenceCommits, undefined);
+    const second = await Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+        ...input,
+        caseId: "second-selection",
+        excludedReferenceCommits: [first.referenceCommit]
+    }));
+    assert.notEqual(second.referenceCommit, first.referenceCommit);
+    assert.ok(f.references.includes(second.referenceCommit));
+    assert.deepEqual(second.excludedReferenceCommits, [first.referenceCommit]);
+    assertRepositoryFoundryGenerationPlanV1(second);
+    const { planDigest: _digest, ...withoutDigest } = second;
+    assert.notEqual(repositoryFoundryGenerationPlanDigestV1({
+        ...withoutDigest,
+        excludedReferenceCommits: ["f".repeat(40)]
+    }), second.planDigest);
+    const invalid = { ...withoutDigest, excludedReferenceCommits: [second.referenceCommit] };
+    assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
+        ...invalid,
+        planDigest: repositoryFoundryGenerationPlanDigestV1(invalid)
+    }), /selected an excluded reference/u);
+    await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+        ...input,
+        caseId: "exhausted",
+        excludedReferenceCommits: [...f.references].sort()
+    })), /no unseen eligible historical seed/u);
+});
+test("reference exclusion inputs are canonical and cannot alter explicit calibration preparation", async () => {
+    const a = "a".repeat(40);
+    const b = "b".repeat(40);
+    assertRepositoryFoundryReferenceExclusionsV1(undefined);
+    assertRepositoryFoundryReferenceExclusionsV1([a, b]);
+    for (const references of [
+        [],
+        [a, a],
+        [b, a],
+        ["A".repeat(40)],
+        ["abc"],
+        ["a".repeat(41)],
+        Array.from({ length: 257 }, (_, index) => index.toString(16).padStart(40, "0"))
+    ]) {
+        assert.throws(() => assertRepositoryFoundryReferenceExclusionsV1(references), /sorted unique lowercase full commit SHAs/u);
+    }
+    await assert.rejects(Effect.runPromise(planHistoricalRepositoryCaseGenerationV1({
+        repositoryRoot: "/never-opened-fixture",
+        referenceCommit: b,
+        excludedReferenceCommits: [a],
+        caseId: "conflicting-calibration",
+        modelPlan
+    })), /generation request failed validation/u);
+});

package/dist/test/repository-seed-selection.test.d.ts

@@ -0,0 +1 @@
+export {};