@velum-labs/routekit-eval-setup 1.0.0

package/dist/test/project-authoring.test.js

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect } from "effect";
+import { readProjectAuthoringSources } from "../project-authoring.js";
+const withRepository = async (use) => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-author-sources-"));
+    const outside = await mkdtemp(path.join(os.tmpdir(), "routekit-author-outside-"));
+    try {
+        await writeFile(path.join(root, "source.md"), "bounded repository source\n");
+        await writeFile(path.join(outside, "secret.md"), "outside\n");
+        await use({ root, outside });
+    }
+    finally {
+        await Promise.all([
+            rm(root, { recursive: true, force: true }),
+            rm(outside, { recursive: true, force: true })
+        ]);
+    }
+};
+const read = (input) => Effect.runPromise(readProjectAuthoringSources(input).pipe(Effect.provide(NodeServicesLayer)));
+test("authoring reads an exact discovered regular source", async () => {
+    await withRepository(async ({ root }) => {
+        assert.deepEqual(await read({
+            repositoryRoot: root,
+            selectedFiles: ["source.md"],
+            sourceInventory: ["source.md"]
+        }), [{ path: "source.md", content: "bounded repository source\n" }]);
+    });
+});
+test("authoring rejects traversal and absolute selected paths", async () => {
+    await withRepository(async ({ root, outside }) => {
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: ["../escape.md"],
+            sourceInventory: ["../escape.md"]
+        }), /canonical relative path/u);
+        const absolute = path.join(outside, "secret.md");
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: [absolute],
+            sourceInventory: [absolute]
+        }), /canonical relative path/u);
+    });
+});
+test("authoring rejects a discovered path replaced by an external symlink", async () => {
+    await withRepository(async ({ root, outside }) => {
+        await symlink(path.join(outside, "secret.md"), path.join(root, "external.md"));
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: ["external.md"],
+            sourceInventory: ["external.md"]
+        }), /regular non-symlink file/u);
+    });
+});
+test("authoring rejects a selected file absent from discovery inventory", async () => {
+    await withRepository(async ({ root }) => {
+        await assert.rejects(read({
+            repositoryRoot: root,
+            selectedFiles: ["source.md"],
+            sourceInventory: []
+        }), /not in the bounded discovery inventory/u);
+    });
+});

package/dist/test/project-workflow.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/project-workflow.test.js

@@ -0,0 +1,516 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect, Layer } from "effect";
+import { EvalRepositoryInspectorLive } from "../inspection.js";
+import { EvalProjectArtifactsLive } from "../project-artifacts.js";
+import { summarizeEvalRunLedger } from "../project-contracts.js";
+import { EvalProjectStoreLive } from "../project-store.js";
+import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const ProjectDependenciesLive = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive, EvalRepositoryInspectorLive).pipe(Layer.provide(NodeServicesLayer));
+const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(NodeServicesLayer));
+const makeRepository = async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-project-"));
+    roots.push(root);
+    await mkdir(path.join(root, "src"), { recursive: true });
+    await writeFile(path.join(root, "README.md"), "# Example application\n");
+    await writeFile(path.join(root, "src", "model.ts"), 'client.responses.create({ model: "openai/current" });\n');
+    return root;
+};
+test("setup creates one durable repository project without invoking model-backed authoring", async () => {
+    const root = await makeRepository();
+    const first = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        return yield* workflow.setup(root);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(first.state.stage, "setup-required");
+    assert.equal(first.question?.id, "workload-description");
+    assert.equal(first.nextAction, "answer");
+    assert.deepEqual(first.state.sourceInventory, ["README.md", "src/model.ts"]);
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        return yield* workflow.setup(root);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(resumed.state.projectId, first.state.projectId);
+    assert.equal(resumed.state.revision, 0);
+    const projectPath = path.join(root, ".routekit", "evals", "project.json");
+    const project = JSON.parse(await readFile(projectPath, "utf8"));
+    assert.equal(project.stage, "setup-required");
+    assert.equal(project.projectId, first.state.projectId);
+    assert.equal((await stat(projectPath)).mode & 0o777, 0o600);
+});
+test("answers advance exactly one question and persist a complete compositional configuration", async () => {
+    const root = await makeRepository();
+    const statuses = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.setup(root);
+        const workload = yield* workflow.answer(root, "Customer support and documentation requests");
+        const candidates = yield* workflow.answer(root, "openai/gpt-5.6-luna, openai/gpt-5.6-terra openai/gpt-5.6-sol");
+        const classifier = yield* workflow.answer(root, "openai/gpt-5.6-luna");
+        const author = yield* workflow.answer(root, "openai/gpt-5.6-terra");
+        const judge = yield* workflow.answer(root, "openai/gpt-5.6-terra");
+        const objective = yield* workflow.answer(root, '{"kind":"highest-quality"}');
+        const unknown = yield* workflow.answer(root, "0.2");
+        const completed = yield* workflow.answer(root, "{}");
+        return { workload, candidates, classifier, author, judge, objective, unknown, completed };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(statuses.workload.question?.id, "candidate-models");
+    assert.equal(statuses.workload.state.revision, 1);
+    assert.equal(statuses.candidates.question?.id, "classifier-model");
+    assert.equal(statuses.classifier.question?.id, "author-model");
+    assert.equal(statuses.author.question?.id, "judge-model");
+    assert.equal(statuses.judge.question?.id, "routing-objective");
+    assert.equal(statuses.objective.question?.id, "maximum-unknown-weight");
+    assert.equal(statuses.unknown.question?.id, "routing-constraints");
+    assert.equal(statuses.completed.state.stage, "dimensions-review");
+    assert.equal(statuses.completed.question, undefined);
+    assert.equal(statuses.completed.nextAction, "propose-dimensions");
+    if (statuses.completed.state.stage !== "dimensions-review") {
+        assert.fail("expected dimensions-review state");
+    }
+    assert.deepEqual(statuses.completed.state.configuration.candidateModels, [
+        "openai/gpt-5.6-luna",
+        "openai/gpt-5.6-terra",
+        "openai/gpt-5.6-sol"
+    ]);
+    assert.equal(statuses.completed.state.configuration.classifierModel, "openai/gpt-5.6-luna");
+    assert.equal(statuses.completed.state.configuration.authorModel, "openai/gpt-5.6-terra");
+    assert.equal(statuses.completed.state.configuration.judgeModel, "openai/gpt-5.6-terra");
+    assert.deepEqual(statuses.completed.state.configuration.objective, {
+        kind: "highest-quality"
+    });
+    assert.equal(statuses.completed.state.configuration.maximumUnknownWeight, 0.2);
+    assert.equal(statuses.completed.state.configuration.constraints, undefined);
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        return yield* workflow.status(root);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(resumed?.state.revision, 8);
+    assert.equal(resumed?.state.stage, "dimensions-review");
+});
+test("invalid answers fail without changing the durable project revision", async () => {
+    const root = await makeRepository();
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.setup(root);
+        const empty = yield* Effect.exit(workflow.answer(root, "   "));
+        assert.equal(empty._tag, "Failure");
+        const current = yield* workflow.status(root);
+        assert.equal(current?.state.revision, 0);
+        assert.equal(current?.question?.id, "workload-description");
+        yield* workflow.answer(root, "Production workload");
+        const duplicateModels = yield* Effect.exit(workflow.answer(root, "openai/one openai/one"));
+        assert.equal(duplicateModels._tag, "Failure");
+        const afterDuplicate = yield* workflow.status(root);
+        assert.equal(afterDuplicate?.state.revision, 1);
+        assert.equal(afterDuplicate?.question?.id, "candidate-models");
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+});
+test("corrupt project state preserves the typed project-store failure", async () => {
+    const root = await makeRepository();
+    await mkdir(path.join(root, ".routekit", "evals"), { recursive: true });
+    await writeFile(path.join(root, ".routekit", "evals", "project.json"), "{not-json\n");
+    const exit = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        return yield* Effect.exit(workflow.status(root));
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(exit._tag, "Failure");
+    if (exit._tag === "Failure") {
+        assert.equal(String(exit.cause).includes("EvalProjectStoreError"), true);
+    }
+});
+const reviewedDimensions = Array.from({ length: 5 }, (_, index) => ({
+    id: `dimension-${String(index + 1)}`,
+    description: `Production workload dimension ${String(index + 1)}`,
+    includes: [`Requests that primarily exercise dimension ${String(index + 1)}`],
+    excludes: [`Requests that primarily exercise another dimension`]
+}));
+const reviewedSuites = reviewedDimensions.map((dimension) => ({
+    version: 1,
+    dimensionId: dimension.id,
+    maximumOutputTokens: 256,
+    cases: Array.from({ length: 20 }, (_, index) => ({
+        id: `${dimension.id}-case-${String(index + 1)}`,
+        prompt: `Solve ${dimension.id} case ${String(index + 1)}`,
+        rubric: `The response must satisfy ${dimension.id} case ${String(index + 1)}`
+    }))
+}));
+const reviewedDecompositionBenchmark = {
+    maximumVectorL1Error: 0.3,
+    cases: Array.from({ length: 20 }, (_, index) => ({
+        id: `decomposition-case-${String(index + 1)}`,
+        request: `Classify production request ${String(index + 1)}`,
+        expected: {
+            weights: reviewedDimensions.map((dimension, dimensionIndex) => ({
+                dimensionId: dimension.id,
+                weight: dimensionIndex === index % reviewedDimensions.length ? 1 : 0
+            })),
+            unknownWeight: 0
+        }
+    }))
+};
+const reviewedCompositionSuite = {
+    maximumOutputTokens: 256,
+    minimumWinnerScoreGap: 0.05,
+    minimumWinnerAgreement: 0.8,
+    cases: Array.from({ length: 20 }, (_, index) => ({
+        id: `composition-case-${String(index + 1)}`,
+        prompt: `Solve multi-dimension production case ${String(index + 1)}`,
+        rubric: `The response must satisfy multi-dimension case ${String(index + 1)}`,
+        decomposition: {
+            weights: reviewedDimensions.map((dimension, dimensionIndex) => ({
+                dimensionId: dimension.id,
+                weight: dimensionIndex === index % reviewedDimensions.length ||
+                    dimensionIndex === (index + 1) % reviewedDimensions.length
+                    ? 0.5
+                    : 0
+            })),
+            unknownWeight: 0
+        },
+        requirements: {
+            endpoint: "responses",
+            requiresTools: false,
+            requiresVision: false
+        }
+    }))
+};
+const reviewedEvaluationInput = {
+    suites: reviewedSuites,
+    decompositionBenchmark: reviewedDecompositionBenchmark,
+    compositionSuite: reviewedCompositionSuite
+};
+async function completeProjectSetup(root) {
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.setup(root);
+        yield* workflow.answer(root, "Production workload");
+        yield* workflow.answer(root, "openai/gpt-5.6-luna openai/gpt-5.6-terra openai/gpt-5.6-sol");
+        yield* workflow.answer(root, "openai/gpt-5.6-luna");
+        yield* workflow.answer(root, "openai/gpt-5.6-terra");
+        yield* workflow.answer(root, "openai/gpt-5.6-terra");
+        yield* workflow.answer(root, '{"kind":"highest-quality"}');
+        yield* workflow.answer(root, "0.2");
+        yield* workflow.answer(root, "{}");
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+}
+test("reviewed artifacts are digest-bound and produce an immutable exact-call plan", async () => {
+    const root = await makeRepository();
+    await completeProjectSetup(root);
+    const result = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposedBasis = yield* workflow.proposeDimensions(root, reviewedDimensions);
+        assert.equal(proposedBasis.nextAction, "approve-dimensions");
+        const basisDigest = proposedBasis.artifacts?.basisProposalDigest;
+        assert.ok(basisDigest !== undefined);
+        const staleBasisApproval = yield* Effect.exit(workflow.approveDimensions(root, "stale-basis-digest"));
+        assert.equal(staleBasisApproval._tag, "Failure");
+        const basisApproved = yield* workflow.approveDimensions(root, basisDigest);
+        assert.equal(basisApproved.state.stage, "evaluations-review");
+        assert.equal(basisApproved.nextAction, "propose-evaluations");
+        const evaluationsProposed = yield* workflow.proposeEvaluations(root, {
+            ...reviewedEvaluationInput
+        });
+        assert.equal(evaluationsProposed.nextAction, "approve-evaluations");
+        const evaluationDigest = evaluationsProposed.artifacts?.evaluationProposalDigest;
+        assert.ok(evaluationDigest !== undefined);
+        const staleEvaluationApproval = yield* Effect.exit(workflow.approveEvaluations(root, "stale-evaluation-digest"));
+        assert.equal(staleEvaluationApproval._tag, "Failure");
+        const ready = yield* workflow.approveEvaluations(root, evaluationDigest);
+        assert.equal(ready.state.stage, "ready");
+        assert.equal(ready.nextAction, "run");
+        const plan = yield* workflow.createPlan(root, "pilot");
+        return { basisDigest, evaluationDigest, plan };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(result.plan.basisDigest, result.basisDigest);
+    assert.equal(result.plan.evaluationDigest, result.evaluationDigest);
+    assert.equal(result.plan.expectedDimensionCandidateCalls, 75);
+    assert.equal(result.plan.expectedDimensionJudgeCalls, 75);
+    assert.equal(result.plan.expectedClassifierCalls, 5);
+    assert.equal(result.plan.expectedCompositionCandidateCalls, 15);
+    assert.equal(result.plan.expectedCompositionJudgeCalls, 15);
+    assert.equal(result.plan.expectedCandidateCalls, 90);
+    assert.equal(result.plan.expectedJudgeCalls, 90);
+    assert.equal(result.plan.expectedCallCount, 185);
+    assert.equal(result.plan.maximumOutputTokens, 256);
+    const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
+    assert.equal((await stat(planPath)).mode & 0o777, 0o600);
+    assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
+    for (const dimension of reviewedDimensions) {
+        assert.equal((await stat(path.join(root, ".routekit", "evals", "dimensions", dimension.id, "suite.json")))
+            .mode & 0o777, 0o600);
+    }
+});
+test("full qualification refuses fewer than twenty reviewed cases per dimension", async () => {
+    const root = await makeRepository();
+    await completeProjectSetup(root);
+    const exit = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposed = yield* workflow.proposeDimensions(root, reviewedDimensions);
+        yield* workflow.approveDimensions(root, proposed.artifacts.basisProposalDigest);
+        const evaluations = yield* workflow.proposeEvaluations(root, {
+            suites: reviewedSuites.map((suite) => ({ ...suite, cases: suite.cases.slice(0, 5) })),
+            decompositionBenchmark: {
+                ...reviewedDecompositionBenchmark,
+                cases: reviewedDecompositionBenchmark.cases.slice(0, 5)
+            },
+            compositionSuite: {
+                ...reviewedCompositionSuite,
+                cases: reviewedCompositionSuite.cases.slice(0, 5)
+            }
+        });
+        yield* workflow.approveEvaluations(root, evaluations.artifacts.evaluationProposalDigest);
+        return yield* Effect.exit(workflow.createPlan(root, "full"));
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(exit._tag, "Failure");
+    if (exit._tag === "Failure") {
+        assert.match(String(exit.cause), /at least 20 reviewed cases/u);
+    }
+});
+const comparisonResults = (plan, runId) => reviewedSuites.map((suite) => {
+    const selected = plan.selectedCaseIds.find((entry) => entry.dimensionId === suite.dimensionId);
+    const cases = suite.cases.filter((testCase) => selected.caseIds.includes(testCase.id));
+    const candidateModels = [
+        "openai/gpt-5.6-luna",
+        "openai/gpt-5.6-terra",
+        "openai/gpt-5.6-sol"
+    ];
+    return {
+        version: 1,
+        comparisonId: `${runId}-${suite.dimensionId}`,
+        profileId: suite.dimensionId,
+        suiteDigest: `suite-${suite.dimensionId}`,
+        judgeModel: "openai/gpt-5.6-terra",
+        startedAt: "2026-08-18T00:00:00.000Z",
+        finishedAt: "2026-08-18T00:01:00.000Z",
+        models: candidateModels.map((model) => ({
+            model,
+            cases: cases.map((testCase) => ({
+                caseId: testCase.id,
+                outcome: "passed",
+                measurement: {
+                    judgeScore: 0.9,
+                    inputTokens: 100,
+                    outputTokens: 50,
+                    durationMs: 250
+                }
+            }))
+        })),
+        calls: [
+            ...candidateModels.flatMap((model) => cases.map((testCase) => ({
+                role: "candidate",
+                model,
+                caseId: testCase.id,
+                measurement: { inputTokens: 100, outputTokens: 50 }
+            }))),
+            ...candidateModels.flatMap(() => cases.map((testCase) => ({
+                role: "judge",
+                model: "openai/gpt-5.6-terra",
+                caseId: testCase.id,
+                measurement: { inputTokens: 80, outputTokens: 20 }
+            })))
+        ]
+    };
+});
+const compositionComparison = (plan, runId) => {
+    const cases = reviewedCompositionSuite.cases.filter((testCase) => plan.selectedCompositionCaseIds.includes(testCase.id));
+    const candidateModels = [...plan.candidateModels];
+    return {
+        version: 1,
+        comparisonId: `${runId}-composition`,
+        profileId: "composition",
+        suiteDigest: "suite-composition",
+        judgeModel: plan.judgeModel,
+        startedAt: "2026-08-18T00:00:00.000Z",
+        finishedAt: "2026-08-18T00:01:00.000Z",
+        models: candidateModels.map((model) => ({
+            model,
+            cases: cases.map((testCase) => ({
+                caseId: testCase.id,
+                outcome: "passed",
+                measurement: {
+                    judgeScore: model === candidateModels[0] ? 0.9 : 0.7,
+                    inputTokens: 100,
+                    outputTokens: 50,
+                    durationMs: 250
+                }
+            }))
+        })),
+        calls: [
+            ...candidateModels.flatMap((model) => cases.map((testCase) => ({
+                role: "candidate",
+                model,
+                caseId: testCase.id,
+                measurement: { inputTokens: 100, outputTokens: 50 }
+            }))),
+            ...candidateModels.flatMap(() => cases.map((testCase) => ({
+                role: "judge",
+                model: plan.judgeModel,
+                caseId: testCase.id,
+                measurement: { inputTokens: 80, outputTokens: 20 }
+            })))
+        ]
+    };
+};
+const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
+    const comparisons = [...comparisonResults(plan, runId), compositionComparison(plan, runId)];
+    const candidateModels = [...plan.candidateModels];
+    const observations = reviewedDecompositionBenchmark.cases
+        .filter((testCase) => plan.selectedDecompositionCaseIds.includes(testCase.id))
+        .map((testCase) => ({
+        caseId: testCase.id,
+        weights: testCase.expected.weights,
+        unknownWeight: testCase.expected.unknownWeight,
+        vectorL1Error: 0,
+        passed: true,
+        measurement: {
+            inputTokens: 40,
+            outputTokens: 20
+        }
+    }));
+    return {
+        version: 1,
+        status: "passed",
+        runId,
+        planId: plan.planId,
+        projectId,
+        startedAt: "2026-08-18T00:00:00.000Z",
+        finishedAt: "2026-08-18T00:02:00.000Z",
+        basisDigest: plan.basisDigest,
+        evaluationDigest: plan.evaluationDigest,
+        target: {
+            kind: publishAllowed ? "configured" : "external",
+            identity: "target:test",
+            publishAllowed
+        },
+        cleanup: publishAllowed
+            ? { sessionOpened: true, sessionClosed: true }
+            : { sessionOpened: false, sessionClosed: false },
+        comparisons,
+        qualification: {
+            decomposition: {
+                expectedCases: observations.length,
+                passedCases: observations.length,
+                maximumObservedL1Error: 0,
+                observations
+            },
+            composition: {
+                expectedCases: plan.selectedCompositionCaseIds.length,
+                comparableCases: plan.selectedCompositionCaseIds.length,
+                agreeingCases: plan.selectedCompositionCaseIds.length,
+                winnerAgreement: 1,
+                cases: plan.selectedCompositionCaseIds.map((caseId) => ({
+                    caseId,
+                    predictedWinner: candidateModels[0],
+                    observedWinner: candidateModels[0],
+                    predictedScoreGap: 0.1,
+                    observedScoreGap: 0.2,
+                    passed: true
+                }))
+            }
+        },
+        ledger: summarizeEvalRunLedger(comparisons, plan.expectedCallCount, observations),
+        activation: {
+            version: 2,
+            generatedAt: "2026-08-18T00:02:00.000Z",
+            basisDigest: plan.basisDigest,
+            evidenceDigest: "evidence-digest",
+            classifierModel: plan.classifierModel,
+            objective: { kind: "highest-quality" },
+            maximumUnknownWeight: 0.2,
+            dimensions: reviewedDimensions,
+            candidateModels,
+            evidence: candidateModels.flatMap((model) => reviewedDimensions.map((dimension) => ({
+                model,
+                dimensionId: dimension.id,
+                suiteDigest: `suite-${dimension.id}`,
+                evidenceDigest: `${model}-${dimension.id}`,
+                quality: {
+                    passRate: 1,
+                    lowerConfidenceBound: 0.56,
+                    sampleCount: plan.scope === "pilot" ? 5 : 20
+                },
+                failureRate: 0,
+                averageJudgeScore: 0.9,
+                p95DurationMs: 250,
+                unpricedCalls: plan.scope === "pilot" ? 5 : 20
+            })))
+        }
+    };
+};
+async function createPlan(root, scope = "pilot") {
+    await completeProjectSetup(root);
+    return Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const basis = yield* workflow.proposeDimensions(root, reviewedDimensions);
+        yield* workflow.approveDimensions(root, basis.artifacts.basisProposalDigest);
+        const evaluations = yield* workflow.proposeEvaluations(root, reviewedEvaluationInput);
+        yield* workflow.approveEvaluations(root, evaluations.artifacts.evaluationProposalDigest);
+        return yield* workflow.createPlan(root, scope);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+}
+test("run lifecycle retains complete sanitized evidence and activates only after cleanup", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "full");
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(root, plan.planId);
+        const current = yield* workflow.status(root);
+        assert.equal(current?.state.stage, "running");
+        const report = qualifiedReport(plan, started.runId, current.state.projectId);
+        const qualified = yield* workflow.finishRun(root, report);
+        const loaded = yield* workflow.result(root, started.runId);
+        const activated = yield* workflow.markActivated(root, started.runId, "target:test");
+        return { started, qualified, loaded, activated };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(outcome.qualified.state.stage, "qualified");
+    assert.equal(outcome.loaded?.status, "passed");
+    assert.equal(outcome.loaded?.ledger.observedCalls, 740);
+    assert.equal(outcome.loaded?.ledger.observedCandidateRows, 360);
+    assert.equal(outcome.loaded?.ledger.unpricedCalls, 740);
+    assert.equal(outcome.activated.state.stage, "activated");
+    const reportPath = path.join(root, ".routekit", "evals", "runs", outcome.started.runId, "report.json");
+    assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
+});
+test("incomplete evidence and external qualification fail closed", async () => {
+    const incompleteRoot = await makeRepository();
+    const incompletePlan = await createPlan(incompleteRoot, "full");
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(incompleteRoot, incompletePlan.planId);
+        const status = yield* workflow.status(incompleteRoot);
+        const report = qualifiedReport(incompletePlan, started.runId, status.state.projectId);
+        assert.equal(report.status, "passed");
+        if (report.status !== "passed")
+            return;
+        const [first, ...rest] = report.comparisons;
+        assert.ok(first !== undefined);
+        const incomplete = {
+            ...report,
+            comparisons: [
+                {
+                    ...first,
+                    models: first.models.map((model, index) => index === 0 ? { ...model, cases: model.cases.slice(0, -1) } : model)
+                },
+                ...rest
+            ]
+        };
+        const exit = yield* Effect.exit(workflow.finishRun(incompleteRoot, incomplete));
+        assert.equal(exit._tag, "Failure");
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    const externalRoot = await makeRepository();
+    const externalPlan = await createPlan(externalRoot, "full");
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(externalRoot, externalPlan.planId);
+        const status = yield* workflow.status(externalRoot);
+        const report = qualifiedReport(externalPlan, started.runId, status.state.projectId, false);
+        yield* workflow.finishRun(externalRoot, report);
+        const exit = yield* Effect.exit(workflow.markActivated(externalRoot, started.runId, "target:test"));
+        assert.equal(exit._tag, "Failure");
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+});

package/dist/test/questions.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/questions.test.js

@@ -0,0 +1,48 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { questionForStage } from "../questions.js";
+test("setup exposes exactly one question for each unresolved stage", () => {
+    const stages = [
+        "surface",
+        "data",
+        "criteria",
+        "constraints",
+        "candidates",
+        "spend-approval",
+        "publish"
+    ];
+    for (const stage of stages) {
+        const question = questionForStage(stage);
+        assert.equal(question?.id, stage);
+        assert.equal(question?.options.length, 3);
+    }
+    assert.equal(questionForStage("completed"), undefined);
+});
+test("repository findings become workspace-specific setup options", () => {
+    const question = questionForStage("surface", {
+        repositoryRoot: "/repo",
+        materials: [],
+        summary: {
+            entriesVisited: 2,
+            textFilesConsidered: 2,
+            filesRead: 2,
+            bytesRead: 100,
+            skippedOversizedFiles: 0,
+            truncated: false
+        },
+        surfaces: [
+            { name: "support", path: "src/support.ts", model: "openai/current" },
+            { name: "triage", path: "src/triage.ts" }
+        ]
+    });
+    assert.deepEqual(question?.options, [
+        "support on openai/current",
+        "triage (src/triage.ts)",
+        "Stop setup"
+    ]);
+});
+test("candidate question requires two candidates and a distinct judge", () => {
+    const question = questionForStage("candidates");
+    assert.match(question?.prompt ?? "", /exactly three unique provider\/model IDs/iu);
+    assert.doesNotMatch(question?.options.join("\n") ?? "", /save without choosing|current model,/iu);
+});

package/dist/test/skill.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/skill.test.js

@@ -0,0 +1,31 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+test("onboarding skill uses the public CLI and preserves approval boundaries", async () => {
+    const skill = await readFile(path.join(packageRoot, "skills", "setup-eval-routing", "SKILL.md"), "utf8");
+    assert.match(skill, /setup-eval-routing/u);
+    assert.match(skill, /one question per turn/iu);
+    assert.match(skill, /Never spend or publish silently/u);
+    assert.match(skill, /public `routekit eval` CLI/u);
+    assert.match(skill, /\$ROUTEKIT eval --help/u);
+    for (const command of ["setup", "status", "answer", "validate", "estimate", "run", "publish"]) {
+        assert.match(skill, new RegExp(`eval ${command}`, "u"));
+    }
+    for (const term of [
+        "routing basis",
+        "workload dimension",
+        "request decomposition",
+        "evidence matrix",
+        "routing activation"
+    ]) {
+        assert.match(skill, new RegExp(term, "iu"));
+    }
+    assert.doesNotMatch(skill, /\beval prepare\b/u);
+    assert.doesNotMatch(skill, /\barea catalog\b/iu);
+    assert.doesNotMatch(skill, /Use RouteKit's `EvalSetup` operations/u);
+    assert.doesNotMatch(skill, /test:e2e:eval-routing/u);
+    assert.doesNotMatch(skill, /OPENROUTER_API_KEY/u);
+});

package/dist/test/state-store.test.d.ts

@@ -0,0 +1 @@
+export {};