@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/project-authoring.test.js

@@ -1,20 +1,21 @@
 import assert from "node:assert/strict";
-import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { Effect, Layer } from "effect";
 import { EvalProjectAuthoringError } from "../errors.js";
-import { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, readProjectAuthoringSources } from "../project-authoring.js";
+import { EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS, EVAL_AUTHORING_CASE_EVIDENCE_BYTES, EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES } from "../evaluation-authoring-policy.js";
+import { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_PREFLIGHT_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, readProjectAuthoringSources, selectProjectAuthoringEvidencePacket, selectProjectAuthoringSourceFiles } from "../project-authoring.js";
 import { EVAL_PROJECT_VERSION } from "../project-contracts.js";
 const withRepository = async (use) => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-author-sources-"));
     const outside = await mkdtemp(path.join(os.tmpdir(), "routekit-author-outside-"));
     try {
-        await writeFile(path.join(root, "source.md"), "bounded repository source\n");
+        await writeFile(path.join(root, "source.md"), "A terminal event must be last; later output is invalid and unsupported.\n");
         await writeFile(path.join(outside, "secret.md"), "outside\n");
-        await use({ root, outside });
+        return await use({ root, outside });
     }
     finally {
         await Promise.all([
@@ -24,6 +25,7 @@ const withRepository = async (use) => {
     }
 };
 const read = (input) => Effect.runPromise(readProjectAuthoringSources(input).pipe(Effect.provide(NodeServicesLayer)));
+const select = (input) => Effect.runPromise(selectProjectAuthoringSourceFiles(input).pipe(Effect.provide(NodeServicesLayer)));
 const configuration = {
     workloadDescription: "Route production requests across separable workload dimensions.",
     candidateModels: ["openai/candidate-a", "openai/candidate-b"],
@@ -60,17 +62,63 @@ const basis = {
     basisDigest: "basis-digest",
     dimensions: dimensions(5)
 };
-const dimensionCases = () => Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => ({
+const promptFor = (index) => {
+    switch (index % 5) {
+        case 0:
+            return `Transform the captured request for scenario ${String(index + 1)}.`;
+        case 1:
+            return `Locate the fault in trace scenario ${String(index + 1)}.`;
+        case 2:
+            return `Construct the expected artifact for scenario ${String(index + 1)}.`;
+        case 3:
+            return `Analyze the boundary behavior in scenario ${String(index + 1)}.`;
+        default:
+            return `Explain the operational outcome for scenario ${String(index + 1)}.`;
+    }
+};
+const authoredCase = (index, evidenceId) => ({
     id: `dimension-case-${String(index + 1)}`,
-    prompt: `Answer production request ${String(index + 1)}.`,
-    context: "Reference context.",
-    rubric: "State the expected production behavior."
-}));
-const dimensionSuite = (dimensionId = basis.dimensions[0].id) => ({
+    prompt: promptFor(index),
+    evidenceIds: [evidenceId],
+    criteria: [
+        {
+            id: `criterion-${String(index + 1)}`,
+            description: `Derive the observable repository behavior for scenario ${String(index + 1)}.`,
+            importance: "critical"
+        }
+    ],
+    positiveControls: [
+        {
+            id: `positive-${String(index + 1)}`,
+            response: `The repository evidence supports the expected scenario ${String(index + 1)} behavior.`
+        }
+    ],
+    negativeControls: [
+        {
+            id: `missing-${String(index + 1)}`,
+            response: "The response discusses the topic but omits the required behavior.",
+            targetedCriterionIds: [`criterion-${String(index + 1)}`],
+            kind: "missing-criterion"
+        },
+        {
+            id: `forbidden-${String(index + 1)}`,
+            response: "The response invents behavior contradicted by the repository evidence.",
+            targetedCriterionIds: [`criterion-${String(index + 1)}`],
+            kind: "forbidden-behavior"
+        },
+        {
+            id: `non-responsive-${String(index + 1)}`,
+            response: "This is a polished overview that never answers the concrete request.",
+            targetedCriterionIds: [`criterion-${String(index + 1)}`],
+            kind: "non-responsive"
+        }
+    ]
+});
+const authoredSuite = (evidenceId, dimensionId = basis.dimensions[0].id) => ({
     version: EVAL_PROJECT_VERSION,
     dimensionId,
     maximumOutputTokens: 256,
-    cases: dimensionCases()
+    cases: Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => authoredCase(index, evidenceId))
 });
 const decompositionWeights = () => basis.dimensions.map((dimension) => ({
     dimensionId: dimension.id,
@@ -87,15 +135,13 @@ const decompositionBenchmark = () => ({
         }
     }))
 });
-const compositionSuite = () => ({
+const authoredCompositionSuite = (evidenceId) => ({
     maximumOutputTokens: 256,
     minimumWinnerScoreGap: 0.05,
     minimumWinnerAgreement: 0.8,
     cases: Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => ({
+        ...authoredCase(index, evidenceId),
         id: `composition-case-${String(index + 1)}`,
-        prompt: `Compose production response ${String(index + 1)}.`,
-        context: "Reference context.",
-        rubric: "State the expected composed production behavior.",
         decomposition: {
             weights: decompositionWeights(),
             unknownWeight: 0
@@ -109,37 +155,242 @@ const compositionSuite = () => ({
         }
     }))
 });
-const proposeEvaluations = (root, outputs, requests = []) => Effect.runPromise(Effect.gen(function* () {
+const authoredPreflight = (packets, replace) => ({
+    cases: packets.map((packet, index) => {
+        const evidenceId = packet.evidenceSources[0].blocks[0].id;
+        return {
+            dimensionId: packet.dimension.id,
+            case: replace?.({ packet, index, evidenceId }) ??
+                authoredCase(index, evidenceId)
+        };
+    })
+});
+const authoredEvaluationCompletion = (input, outputs, requests) => {
+    requests.push(input);
+    const authoringInput = JSON.parse(input.input);
+    const evidenceId = authoringInput.evidenceSources?.[0]?.blocks?.[0]?.id;
+    if (input.schemaName !== "routekit_decomposition_benchmark" &&
+        input.schemaName !== "routekit_evaluation_authoring_preflight") {
+        assert.ok(evidenceId);
+    }
+    const withCallId = (text) => ({
+        text,
+        ...(outputs.callId === undefined ? {} : { callId: outputs.callId })
+    });
+    if (input.schemaName === "routekit_evaluation_authoring_preflight") {
+        const packets = (authoringInput.packets ?? []).map((packet) => {
+            const dimensionId = packet.dimension?.id;
+            const packetEvidenceId = packet.evidenceSources?.[0]?.blocks?.[0]?.id;
+            assert.ok(dimensionId);
+            assert.ok(packetEvidenceId);
+            return {
+                dimension: { id: dimensionId },
+                evidenceSources: [{ blocks: [{ id: packetEvidenceId }] }]
+            };
+        });
+        return withCallId(JSON.stringify(typeof outputs.preflight === "function"
+            ? outputs.preflight(packets)
+            : outputs.preflight ?? authoredPreflight(packets)));
+    }
+    if (input.schemaName === "routekit_dimension_suite") {
+        const dimension = basis.dimensions.find((candidate) => input.operationId.endsWith(`:${candidate.id}`));
+        const dimensionId = dimension?.id ?? basis.dimensions[0].id;
+        return withCallId(JSON.stringify(typeof outputs.suite === "function"
+            ? outputs.suite(evidenceId, dimensionId)
+            : outputs.suite ?? authoredSuite(evidenceId, dimensionId)));
+    }
+    if (input.schemaName === "routekit_decomposition_benchmark") {
+        return withCallId(JSON.stringify(outputs.decomposition ?? decompositionBenchmark()));
+    }
+    return withCallId(JSON.stringify(typeof outputs.composition === "function"
+        ? outputs.composition(evidenceId)
+        : outputs.composition ?? authoredCompositionSuite(evidenceId)));
+};
+const proposeEvaluations = (root, outputs, requests = [], authoringBasis = basis, sourceInventory = ["source.md"]) => Effect.runPromise(Effect.gen(function* () {
     const author = yield* EvalProjectAuthor;
     return yield* author.proposeEvaluations({
         operationId: "eng-833",
         repositoryRoot: root,
-        sourceInventory: ["source.md"],
+        sourceInventory,
         configuration,
-        basis
+        basis: authoringBasis
     });
 }).pipe(Effect.provide(EvalProjectAuthorLive.pipe(Layer.provide(Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
-    complete: (input) => Effect.sync(() => {
-        requests.push(input);
-        if (input.schemaName === "routekit_dimension_suite") {
-            const dimension = basis.dimensions.find((candidate) => input.operationId.endsWith(`:${candidate.id}`));
-            return JSON.stringify(outputs.suite ?? dimensionSuite(dimension?.id ?? basis.dimensions[0].id));
-        }
-        if (input.schemaName === "routekit_decomposition_benchmark") {
-            return JSON.stringify(outputs.decomposition ?? decompositionBenchmark());
-        }
-        return JSON.stringify(outputs.composition ?? compositionSuite());
-    })
+    complete: (input) => Effect.sync(() => authoredEvaluationCompletion(input, outputs, requests).text),
+    completeDetailed: (input) => Effect.sync(() => authoredEvaluationCompletion(input, outputs, requests))
 }))), Layer.provide(NodeServicesLayer)))));
-test("evaluation authoring requires contexts to carry protocol-specific rubric facts", async () => {
+test("evaluation authoring compiles opaque evidence selections and hidden controls", async () => {
     const requests = [];
-    await withRepository(async ({ root }) => {
-        await proposeEvaluations(root, {}, requests);
+    const proposal = await withRepository(async ({ root }) => {
+        return proposeEvaluations(root, {}, requests);
     });
     const suiteRequest = requests.find((request) => request.schemaName === "routekit_dimension_suite");
     assert.ok(suiteRequest);
-    assert.match(suiteRequest.instructions, /context must state every protocol-specific envelope, field, terminal event, mapping, or invariant/u);
-    assert.match(suiteRequest.instructions, /do not assume unstated wire-format knowledge/u);
+    assert.match(suiteRequest.instructions, new RegExp(`Select at most ${String(EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS)} evidenceIds per case`, "u"));
+    assert.match(suiteRequest.instructions, /Do not produce paths, line ranges, excerpts/u);
+    assert.match(suiteRequest.instructions, /hidden criteria and controls/u);
+    const suiteCaseSchema = suiteRequest.jsonSchema.properties.cases.items.properties;
+    assert.equal(Object.hasOwn(suiteCaseSchema, "context"), false);
+    assert.equal(Object.hasOwn(suiteCaseSchema, "rubric"), false);
+    const suiteInput = JSON.parse(suiteRequest.input);
+    const suppliedEvidenceIds = suiteInput.evidenceSources.flatMap((source) => source.blocks.map((block) => block.id));
+    assert.deepEqual(suiteCaseSchema.evidenceIds, {
+        type: "array",
+        minItems: 1,
+        maxItems: EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS,
+        items: { type: "string", enum: suppliedEvidenceIds }
+    });
+    const firstCase = proposal.suites[0].cases[0];
+    assert.equal(firstCase.evidenceIds?.length, 1);
+    assert.match(firstCase.evidenceIds[0], /^ev_[0-9a-f]{24}$/u);
+    assert.match(firstCase.context ?? "", /must be last/u);
+    assert.match(firstCase.context ?? "", /invalid and unsupported/u);
+    assert.match(firstCase.rubric, /Critical criterion criterion-1/u);
+    assert.equal(firstCase.criteria?.[0]?.gradingMode, "semantic");
+    assert.deepEqual(firstCase.negativeControls?.map((control) => control.kind), ["missing-criterion", "forbidden-behavior", "non-responsive"]);
+    assert.ok(proposal.evidenceSources !== undefined);
+});
+test("evaluation authoring preflight rejects unknown evidence before full-suite spending", async () => {
+    const requests = [];
+    await withRepository(async ({ root }) => {
+        await assert.rejects(proposeEvaluations(root, {
+            callId: "model_call_cross_dimension_preflight",
+            preflight: (packets) => authoredPreflight(packets, ({ index, evidenceId }) => authoredCase(index, index === 0 ? "ev_unknown" : evidenceId))
+        }, requests), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(error.detail, /cross-dimension authoring preflight failed validation/u);
+            assert.match(error.detail, /call id model_call_cross_dimension_preflight/u);
+            assert.match(error.detail, /category evidence-selection/u);
+            assert.match(error.detail, /path \$\.cases\[0\]\.case\.evidenceIds/u);
+            assert.match(error.detail, /unknown evidence id/u);
+            return true;
+        });
+    });
+    assert.deepEqual(requests.map((request) => request.schemaName), ["routekit_evaluation_authoring_preflight"]);
+});
+test("evaluation authoring preflight rejects malformed grader controls", async () => {
+    const requests = [];
+    await withRepository(async ({ root }) => {
+        await assert.rejects(proposeEvaluations(root, {
+            preflight: (packets) => authoredPreflight(packets, ({ index, evidenceId }) => {
+                const testCase = authoredCase(index, evidenceId);
+                return index === packets.length - 1
+                    ? {
+                        ...testCase,
+                        negativeControls: [testCase.negativeControls[0]]
+                    }
+                    : testCase;
+            })
+        }, requests), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(error.detail, /cross-dimension authoring preflight failed validation/u);
+            assert.match(error.detail, /path \$\.cases\[4\]\.case\.negativeControls/u);
+            assert.match(error.detail, /requires a forbidden-behavior negative control/u);
+            return true;
+        });
+    });
+    assert.deepEqual(requests.map((request) => request.schemaName), ["routekit_evaluation_authoring_preflight"]);
+});
+test("cross-dimension preflight requires every supplied dimension exactly once", async () => {
+    await withRepository(async ({ root }) => {
+        for (const scenario of [
+            {
+                label: "missing",
+                output: (packets) => ({
+                    cases: authoredPreflight(packets).cases.slice(1)
+                }),
+                expected: /exactly one case per supplied dimension/u
+            },
+            {
+                label: "duplicate",
+                output: (packets) => {
+                    const authored = authoredPreflight(packets);
+                    return {
+                        cases: [
+                            authored.cases[0],
+                            {
+                                ...authored.cases[1],
+                                dimensionId: authored.cases[0].dimensionId
+                            },
+                            ...authored.cases.slice(2)
+                        ]
+                    };
+                },
+                expected: /must be unique/u
+            },
+            {
+                label: "unknown",
+                output: (packets) => {
+                    const authored = authoredPreflight(packets);
+                    return {
+                        cases: [
+                            {
+                                ...authored.cases[0],
+                                dimensionId: "dimension-unknown"
+                            },
+                            ...authored.cases.slice(1)
+                        ]
+                    };
+                },
+                expected: /must identify a supplied routing dimension/u
+            }
+        ]) {
+            const requests = [];
+            await assert.rejects(proposeEvaluations(root, { preflight: scenario.output }, requests), (error) => {
+                assert.ok(error instanceof EvalProjectAuthoringError);
+                assert.match(error.detail, scenario.expected, scenario.label);
+                return true;
+            });
+            assert.deepEqual(requests.map((request) => request.schemaName), ["routekit_evaluation_authoring_preflight"]);
+        }
+    });
+});
+test("cross-dimension preflight rejects evidence from another dimension packet", async () => {
+    const requests = [];
+    await withRepository(async ({ root }) => {
+        const separatedDimensions = [
+            "alphaowner",
+            "bravoowner",
+            "charlieowner",
+            "deltaowner",
+            "echoowner"
+        ].map((term, index) => ({
+            ...dimensions(5)[index],
+            id: `${term}-dimension`,
+            description: `Production behavior owned by ${term}.`,
+            includes: [term]
+        }));
+        const separatedBasis = {
+            version: 2,
+            basisDigest: "separated-basis-digest",
+            dimensions: separatedDimensions
+        };
+        const sourceInventory = separatedDimensions.map((dimension) => `${dimension.id}.md`);
+        await Promise.all(separatedDimensions.map((dimension, index) => writeFile(path.join(root, sourceInventory[index]), `${dimension.id}\n`)));
+        await assert.rejects(proposeEvaluations(root, {
+            preflight: (packets) => {
+                const authored = authoredPreflight(packets);
+                const foreignEvidenceId = packets[1].evidenceSources[0].blocks[0].id;
+                return {
+                    cases: authored.cases.map((item, index) => index === 0
+                        ? {
+                            ...item,
+                            case: {
+                                ...item.case,
+                                evidenceIds: [foreignEvidenceId]
+                            }
+                        }
+                        : item)
+                };
+            }
+        }, requests, separatedBasis, sourceInventory), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(error.detail, /category evidence-selection/u);
+            assert.match(error.detail, /unknown evidence id/u);
+            return true;
+        });
+    });
+    assert.deepEqual(requests.map((request) => request.schemaName), ["routekit_evaluation_authoring_preflight"]);
 });
 const collectSchemaNumberKeyword = (value, keyword) => {
     if (typeof value !== "object" || value === null)
@@ -159,7 +410,12 @@ test("authoring reads an exact discovered regular source", async () => {
             repositoryRoot: root,
             selectedFiles: ["source.md"],
             sourceInventory: ["source.md"]
-        }), [{ path: "source.md", content: "bounded repository source\n" }]);
+        }), [
+            {
+                path: "source.md",
+                content: "A terminal event must be last; later output is invalid and unsupported.\n"
+            }
+        ]);
     });
 });
 test("authoring rejects traversal and absolute selected paths", async () => {
@@ -196,6 +452,221 @@ test("authoring rejects a selected file absent from discovery inventory", async
         }), /not in the bounded discovery inventory/u);
     });
 });
+test("authoring preserves selected-source file and byte caps", async () => {
+    await withRepository(async ({ root }) => {
+        const files = Array.from({ length: EVAL_AUTHORING_SOURCE_FILES + 1 }, (_, index) => `src/file-${String(index).padStart(2, "0")}.ts`);
+        await mkdir(path.join(root, "src"), { recursive: true });
+        await Promise.all(files.map((relative) => writeFile(path.join(root, relative), "export {};\n")));
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: files,
+            sourceInventory: files
+        }), /select between 1 and 64 discovered source files/u);
+        await writeFile(path.join(root, "src/large-a.ts"), "a".repeat(30_001));
+        await writeFile(path.join(root, "src/large-b.ts"), "b".repeat(30_000));
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: ["src/large-a.ts", "src/large-b.ts"],
+            sourceInventory: ["src/large-a.ts", "src/large-b.ts"]
+        }), /exceed the 60000 byte authoring bound/u);
+    });
+});
+test("authoring source selection is independent of inventory order", async () => {
+    await withRepository(async ({ root }) => {
+        const sources = [
+            ["src/router.ts", "export const routeProductionRequest = () => \"routed\";\n"],
+            ["test/router.test.ts", "test(\"routes a production request\", () => {});\n"],
+            ["src/protocol/request-contract.ts", "export type RouteRequest = { model: string };\n"],
+            ["docs/routing.md", "# Production routing\n"],
+            ["scripts/verify-routing.ts", "export const verifyRouting = true;\n"]
+        ];
+        for (const [relative, content] of sources) {
+            await mkdir(path.dirname(path.join(root, relative)), { recursive: true });
+            await writeFile(path.join(root, relative), content);
+        }
+        const inventory = sources.map(([relative]) => relative);
+        const selectionInput = {
+            repositoryRoot: root,
+            workloadDescription: "Route production requests across provider protocols.",
+            targetDimensions: basis.dimensions
+        };
+        const expected = await select({ ...selectionInput, sourceInventory: inventory });
+        const shuffled = await select({
+            ...selectionInput,
+            sourceInventory: [inventory[3], inventory[0], inventory[4], inventory[1], inventory[2]]
+        });
+        assert.deepEqual(shuffled, expected);
+        assert.deepEqual([...expected].sort(), expected);
+    });
+});
+test("authoring drops leading session metadata and keeps relevant evidence within bounds", async () => {
+    await withRepository(async ({ root }) => {
+        const metadata = Array.from({ length: 20 }, (_, index) => `.ori/logs/sessions/session-${String(index).padStart(2, "0")}/metadata.json`);
+        const relevant = [
+            "packages/gateway/src/routing/provider-protocol.ts",
+            "packages/gateway/src/test/provider-protocol.test.ts",
+            "packages/contracts/src/provider-protocol-contract.ts",
+            "docs/provider-routing.md",
+            "scripts/verify-provider-routing.ts"
+        ];
+        for (const relative of metadata) {
+            await mkdir(path.dirname(path.join(root, relative)), { recursive: true });
+            await writeFile(path.join(root, relative), "m".repeat(8_000));
+        }
+        for (const relative of relevant) {
+            await mkdir(path.dirname(path.join(root, relative)), { recursive: true });
+            await writeFile(path.join(root, relative), `${relative}\n${"r".repeat(9_000)}`);
+        }
+        const selected = await select({
+            repositoryRoot: root,
+            sourceInventory: [...metadata, ...relevant],
+            workloadDescription: "Route provider protocol requests.",
+            targetDimensions: [
+                {
+                    id: "provider-protocol",
+                    description: "Translate provider protocol request and response envelopes",
+                    includes: ["provider protocol routing"],
+                    excludes: ["unrelated requests"],
+                    inScopeRequest: "Translate a provider protocol request.",
+                    nearMissRequest: "Handle an unrelated request."
+                }
+            ]
+        });
+        const selectedBytes = (await Promise.all(selected.map(async (relative) => Buffer.byteLength(await readFile(path.join(root, relative), "utf8"))))).reduce((total, bytes) => total + bytes, 0);
+        assert.ok(relevant.every((relative) => selected.includes(relative)));
+        assert.ok(selected.every((relative) => !relative.startsWith(".ori/")));
+        assert.ok(selected.length <= EVAL_AUTHORING_SOURCE_FILES);
+        assert.ok(selectedBytes <= EVAL_AUTHORING_SOURCE_BYTES);
+    });
+});
+test("authoring keeps implementation paths containing session, cache, or log segments", async () => {
+    await withRepository(async ({ root }) => {
+        const implementation = [
+            "packages/gateway/src/session/handler.ts",
+            "packages/gateway/src/cache/store.ts",
+            "packages/gateway/src/log/writer.ts"
+        ];
+        for (const relative of implementation) {
+            await mkdir(path.dirname(path.join(root, relative)), { recursive: true });
+            await writeFile(path.join(root, relative), `export const source = ${JSON.stringify(relative)};\n`);
+        }
+        const selected = await select({
+            repositoryRoot: root,
+            sourceInventory: implementation,
+            workloadDescription: "Route gateway session requests with cached logging state."
+        });
+        assert.deepEqual(selected, [...implementation].sort());
+    });
+});
+test("authoring records a versioned selected-source manifest", async () => {
+    await withRepository(async ({ root }) => {
+        const sourceInventory = [
+            "src/router.ts",
+            "test/router.test.ts",
+            "src/protocol/request-contract.ts",
+            "docs/routing.md",
+            "scripts/verify-routing.ts"
+        ];
+        for (const relative of sourceInventory) {
+            await mkdir(path.dirname(path.join(root, relative)), { recursive: true });
+            await writeFile(path.join(root, relative), `${relative}\n`);
+        }
+        const requests = [];
+        await proposeDimensions(root, { dimensions: dimensions(5) }, requests, sourceInventory);
+        const manifest = JSON.parse(await readFile(path.join(root, ".routekit/evals/authoring-sources.dimensions.v1.json"), "utf8"));
+        assert.equal(manifest.version, 1);
+        assert.equal(manifest.purpose, "dimensions");
+        assert.ok(manifest.packets[0].sources.length > 0);
+        assert.ok(manifest.packets[0].sources.every((source) => typeof source.path === "string" &&
+            typeof source.sizeBytes === "number" &&
+            typeof source.sourceClass === "string" &&
+            Array.isArray(source.targetDimensions) &&
+            typeof source.inclusionReason === "string"));
+        assert.ok(manifest.packets[0].sources.every((source) => JSON.stringify(Object.keys(source).sort()) ===
+            JSON.stringify([
+                "inclusionReason",
+                "path",
+                "sizeBytes",
+                "sourceClass",
+                "targetDimensions"
+            ])));
+    });
+});
+test("authoring only admits excluded artifacts with an explicit recorded reason", async () => {
+    await withRepository(async ({ root }) => {
+        const metadata = ".ori/logs/sessions/session-01/metadata.json";
+        await mkdir(path.dirname(path.join(root, metadata)), { recursive: true });
+        await writeFile(path.join(root, metadata), "session evidence\n");
+        assert.deepEqual(await select({
+            repositoryRoot: root,
+            sourceInventory: [metadata, "source.md"],
+            workloadDescription: "Route production requests."
+        }), ["source.md"]);
+        assert.deepEqual(await Effect.runPromise(selectProjectAuthoringSourceFiles({
+            repositoryRoot: root,
+            sourceInventory: [metadata, "source.md"],
+            workloadDescription: "Route production requests.",
+            explicitInclusions: [
+                { path: metadata, reason: "incident-specific operational evidence" }
+            ]
+        }).pipe(Effect.provide(NodeServicesLayer))), [metadata, "source.md"]);
+        await assert.rejects(Effect.runPromise(selectProjectAuthoringSourceFiles({
+            repositoryRoot: root,
+            sourceInventory: [metadata, "source.md"],
+            workloadDescription: "Route production requests.",
+            explicitInclusions: [{ path: metadata, reason: " " }]
+        }).pipe(Effect.provide(NodeServicesLayer))), /requires a recorded inclusion reason/u);
+    });
+});
+test("evaluation evidence packets select compiler blocks instead of whole files", async () => {
+    await withRepository(async ({ root }) => {
+        const relevant = "packages/gateway/src/providers/openai-backend.ts";
+        const broad = "packages/gateway/src/test/router.test.ts";
+        const historical = "docs/evidence/eval-routing/old/published-routing.json";
+        for (const [relative, content] of [
+            [
+                relevant,
+                [
+                    "export const executeProviderModel = true;\n",
+                    "\n",
+                    "export const decodeProviderResponse = true;\n",
+                    "\n",
+                    "export const unrelated = true;\n"
+                ].join("")
+            ],
+            [
+                broad,
+                Array.from({ length: 80 }, (_, index) => `test provider request response routing ${String(index)};\n`).join("")
+            ],
+            [historical, '{"provider":"historical routing result"}\n']
+        ]) {
+            await mkdir(path.dirname(path.join(root, relative)), {
+                recursive: true
+            });
+            await writeFile(path.join(root, relative), content);
+        }
+        const result = await Effect.runPromise(selectProjectAuthoringEvidencePacket({
+            repositoryRoot: root,
+            sourceInventory: [relevant, broad, historical],
+            targetDimensions: [
+                {
+                    id: "provider-model-execution",
+                    description: "Execute explicit provider model requests",
+                    includes: [
+                        "provider backend request execution",
+                        "provider response decoding"
+                    ],
+                    excludes: ["semantic routing"]
+                }
+            ]
+        }).pipe(Effect.provide(NodeServicesLayer)));
+        assert.ok(result.evidenceSources.some((source) => source.path === relevant));
+        assert.ok(result.evidenceSources.every((source) => source.path !== historical));
+        assert.ok(result.evidenceSources.every((source) => source.sourceBlockCount !== undefined &&
+            source.blocks.length <= source.sourceBlockCount));
+        assert.ok(result.packet.sources.reduce((total, source) => total + source.sizeBytes, 0) <= EVAL_AUTHORING_SOURCE_BYTES);
+    });
+});
 test("dimension authoring sends an Anthropic-compatible structured output schema", async () => {
     await withRepository(async ({ root }) => {
         const requests = [];
@@ -322,7 +793,10 @@ test("evaluation authoring enforces Anthropic-deferred bounds after parsing", as
     await withRepository(async ({ root }) => {
         for (const maximumOutputTokens of [0, 16_385]) {
             await assert.rejects(proposeEvaluations(root, {
-                suite: { ...dimensionSuite(), maximumOutputTokens }
+                suite: (evidenceId, dimensionId) => ({
+                    ...authoredSuite(evidenceId, dimensionId),
+                    maximumOutputTokens
+                })
             }), (error) => {
                 assert.ok(error instanceof EvalProjectAuthoringError);
                 assert.match(error.detail, /suite .* failed validation/u);
@@ -331,9 +805,30 @@ test("evaluation authoring enforces Anthropic-deferred bounds after parsing", as
             });
         }
         await assert.rejects(proposeEvaluations(root, {
-            suite: {
-                ...dimensionSuite(),
-                cases: dimensionCases().slice(0, EVAL_AUTHORING_CASES_PER_DIMENSION - 1)
+            suite: (evidenceId, dimensionId) => {
+                const suite = authoredSuite(evidenceId, dimensionId);
+                return {
+                    ...suite,
+                    cases: suite.cases.map((testCase, index) => index === 0
+                        ? {
+                            ...testCase,
+                            evidenceIds: Array.from({ length: EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS + 1 }, () => evidenceId)
+                        }
+                        : testCase)
+                };
+            }
+        }), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(String(error.cause), new RegExp(`evidenceIds.*must contain at most ${String(EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS)} items`, "u"));
+            return true;
+        });
+        await assert.rejects(proposeEvaluations(root, {
+            suite: (evidenceId, dimensionId) => {
+                const suite = authoredSuite(evidenceId, dimensionId);
+                return {
+                    ...suite,
+                    cases: suite.cases.slice(0, EVAL_AUTHORING_CASES_PER_DIMENSION - 1)
+                };
             }
         }), (error) => {
             assert.ok(error instanceof EvalProjectAuthoringError);
@@ -349,7 +844,10 @@ test("evaluation authoring enforces Anthropic-deferred bounds after parsing", as
             return true;
         });
         await assert.rejects(proposeEvaluations(root, {
-            composition: { ...compositionSuite(), minimumWinnerAgreement: 2 }
+            composition: (evidenceId) => ({
+                ...authoredCompositionSuite(evidenceId),
+                minimumWinnerAgreement: 2
+            })
         }), (error) => {
             assert.ok(error instanceof EvalProjectAuthoringError);
             assert.equal(error.detail, "composition benchmark failed validation");
@@ -358,14 +856,66 @@ test("evaluation authoring enforces Anthropic-deferred bounds after parsing", as
         });
     });
 });
+test("evaluation authoring exposes bounded call-linked validation diagnostics", async () => {
+    await withRepository(async ({ root }) => {
+        await assert.rejects(proposeEvaluations(root, {
+            callId: "model_call_authoring_validation",
+            suite: (evidenceId, dimensionId) => {
+                const suite = authoredSuite(evidenceId, dimensionId);
+                return {
+                    ...suite,
+                    cases: suite.cases.map((testCase, index) => index === 0
+                        ? {
+                            ...testCase,
+                            criteria: testCase.criteria.map((criterion) => ({
+                                ...criterion,
+                                importance: "supporting"
+                            }))
+                        }
+                        : testCase)
+                };
+            }
+        }), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(error.detail, /call id model_call_authoring_validation/u);
+            assert.match(error.detail, /category criterion-topology/u);
+            assert.match(error.detail, /path \$\.cases\[0\]\.criteria/u);
+            assert.match(error.detail, /requires at least one critical criterion/u);
+            assert.doesNotMatch(error.detail, /Known-good response/u);
+            return true;
+        });
+    });
+});
 test("evaluation authoring budgets enough output for all twenty requested cases", async () => {
     await withRepository(async ({ root }) => {
+        assert.equal(EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS *
+            EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES, EVAL_AUTHORING_CASE_EVIDENCE_BYTES);
+        assert.ok((EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS + 1) *
+            EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES >
+            EVAL_AUTHORING_CASE_EVIDENCE_BYTES);
         const requests = [];
         const proposal = await proposeEvaluations(root, {}, requests);
         assert.equal(proposal.suites.length, basis.dimensions.length);
-        assert.equal(requests.length, basis.dimensions.length + 2);
-        assert.ok(requests.every((request) => request.maximumOutputTokens === EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS));
-        assert.ok(requests.every((request) => request.instructions.includes("exactly 20") &&
+        assert.equal(requests.length, basis.dimensions.length + 3);
+        const preflightRequests = requests.filter((request) => request.schemaName === "routekit_evaluation_authoring_preflight");
+        assert.equal(preflightRequests.length, 1);
+        const preflight = preflightRequests[0];
+        assert.equal(preflight.schemaName, "routekit_evaluation_authoring_preflight");
+        assert.equal(preflight.maximumOutputTokens, EVAL_AUTHORING_PREFLIGHT_OUTPUT_TOKENS);
+        assert.ok(Buffer.byteLength(preflight.input, "utf8") <
+            EVAL_AUTHORING_REQUEST_BYTES);
+        const preflightInput = JSON.parse(preflight.input);
+        assert.deepEqual(preflightInput.packets.map((packet) => packet.dimension.id), basis.dimensions.map((dimension) => dimension.id));
+        assert.ok(preflightInput.packets.every((packet) => packet.evidenceSources.reduce((total, source) => total + source.blocks.length, 0) <= EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS));
+        const preflightCasesSchema = preflight.jsonSchema.properties.cases;
+        assert.equal(preflightCasesSchema.minItems, basis.dimensions.length);
+        assert.equal(preflightCasesSchema.maxItems, basis.dimensions.length);
+        assert.deepEqual(preflightCasesSchema.items.anyOf.map((variant) => variant.properties.dimensionId.enum[0]), basis.dimensions.map((dimension) => dimension.id));
+        assert.deepEqual(preflightCasesSchema.items.anyOf.map((variant) => variant.properties.case.properties.evidenceIds.items.enum), preflightInput.packets.map((packet) => packet.evidenceSources.flatMap((source) => source.blocks.map((block) => block.id))));
+        assert.ok(requests.slice(1).every((request) => request.maximumOutputTokens === EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS));
+        assert.ok(requests
+            .filter((request) => request.schemaName !== "routekit_evaluation_authoring_preflight")
+            .every((request) => request.instructions.includes("exactly 20") &&
             request.instructions.includes("Keep each case concise")));
     });
 });