@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/evaluation-grader-calibration.test.js

@@ -0,0 +1,373 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { assertEvaluationGraderCalibrationPassed, compileEvaluationGraderCalibrationPlan, compileEvaluationGraderCalibrationReport, evaluationGraderCalibrationReportDigest } from "../evaluation-grader-calibration.js";
+import { EVAL_GRADER_MINIMUM_SCORE, EVAL_GRADER_SYSTEM_PROMPT, renderEvaluationCandidatePrompt, renderEvaluationCandidatePromptSource } from "../evaluation-grading-policy.js";
+import { compileEvaluationEvidence, renderEvaluationEvidenceContext } from "../evaluation-evidence.js";
+import { evaluationProposalDigest } from "../evaluation-proposal-policy.js";
+import { renderEvaluationCriteria } from "../evaluation-structure-policy.js";
+const criteria = [
+    {
+        id: "critical",
+        description: "State that the terminal event must be last.",
+        importance: "critical",
+        gradingMode: "semantic"
+    }
+];
+const evidence = compileEvaluationEvidence({
+    sources: [
+        {
+            path: "src/protocol.ts",
+            content: "The terminal event must be emitted last.\n"
+        }
+    ],
+    maximumBlockBytes: 3_000
+});
+const evidenceBlock = evidence[0].blocks[0];
+const compiledCase = (id) => ({
+    id,
+    prompt: "Explain the required terminal event ordering.",
+    context: renderEvaluationEvidenceContext([evidenceBlock]),
+    rubric: renderEvaluationCriteria(criteria),
+    evidenceIds: [evidenceBlock.id],
+    criteria,
+    positiveControls: [
+        {
+            id: "positive",
+            response: "The terminal event is emitted last."
+        }
+    ],
+    negativeControls: [
+        {
+            id: "missing",
+            response: "The response discusses events without stating their order.",
+            targetedCriterionIds: ["critical"],
+            kind: "missing-criterion"
+        },
+        {
+            id: "forbidden",
+            response: "Emit another data event after the terminal event.",
+            targetedCriterionIds: ["critical"],
+            kind: "forbidden-behavior"
+        },
+        {
+            id: "non-answer",
+            response: "Event ordering is an important topic with many nuances.",
+            targetedCriterionIds: ["critical"],
+            kind: "non-responsive"
+        }
+    ]
+});
+const proposalPayload = () => ({
+    version: 1,
+    basisDigest: "basis-digest",
+    candidateModels: ["provider/a", "provider/b"],
+    judgeModel: "provider/judge",
+    evidenceSources: evidence,
+    suites: [
+        {
+            version: 1,
+            dimensionId: "protocol",
+            maximumOutputTokens: 1_024,
+            cases: Array.from({ length: 5 }, (_, index) => compiledCase(`dimension-case-${String(index + 1)}`))
+        }
+    ],
+    decompositionBenchmark: {
+        maximumVectorL1Error: 0.1,
+        cases: Array.from({ length: 5 }, (_, index) => ({
+            id: `decomposition-${String(index + 1)}`,
+            request: `Classify request ${String(index + 1)}.`,
+            expected: {
+                weights: [{ dimensionId: "protocol", weight: 1 }],
+                unknownWeight: 0
+            }
+        }))
+    },
+    compositionSuite: {
+        maximumOutputTokens: 2_048,
+        minimumWinnerScoreGap: 0.1,
+        minimumWinnerAgreement: 0.8,
+        cases: Array.from({ length: 5 }, (_, index) => ({
+            ...compiledCase(`composition-case-${String(index + 1)}`),
+            decomposition: {
+                weights: [{ dimensionId: "protocol", weight: 1 }],
+                unknownWeight: 0
+            },
+            requirements: {
+                endpoint: "responses",
+                requiresTools: false,
+                requiresVision: false
+            }
+        }))
+    }
+});
+const proposalFrom = (payload) => ({
+    ...payload,
+    evaluationDigest: evaluationProposalDigest(payload)
+});
+const proposal = () => proposalFrom(proposalPayload());
+const sourceInventory = ["src/protocol.ts"];
+const observationsFor = (plan) => plan.controls.map((control, index) => ({
+    controlId: control.id,
+    judgeModel: plan.judgeModel,
+    callId: `model_call_${String(index + 1)}`,
+    pass: control.expectedOutcome === "accept",
+    score: control.expectedOutcome === "accept"
+        ? EVAL_GRADER_MINIMUM_SCORE
+        : EVAL_GRADER_MINIMUM_SCORE - 0.1,
+    reason: control.expectedOutcome === "accept"
+        ? "The response satisfies the criterion."
+        : "The response does not satisfy the criterion.",
+    durationMs: 10
+}));
+test("candidate prompt source and pure compiler renderer implement the same bytes", () => {
+    const candidateCase = {
+        prompt: "Explain the required terminal event ordering.",
+        context: renderEvaluationEvidenceContext([evidenceBlock])
+    };
+    const generated = new Function("candidateCase", `${renderEvaluationCandidatePromptSource()}\nreturn prompt;`);
+    assert.equal(generated(candidateCase), renderEvaluationCandidatePrompt(candidateCase));
+});
+test("grader calibration compiles every authored control without category sampling", () => {
+    const plan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposal(),
+        sourceInventory
+    });
+    assert.equal(plan.policyVersion, 1);
+    assert.equal(plan.judgeProtocolVersion, 1);
+    assert.equal(plan.judgeReasoningEffort, null);
+    assert.equal(plan.judgeSystemPrompt, EVAL_GRADER_SYSTEM_PROMPT);
+    assert.equal(plan.minimumJudgeScore, 0.8);
+    assert.equal(plan.controls.length, 40);
+    assert.equal(plan.callCount, 40);
+    assert.equal(new Set(plan.controls.map((control) => control.id)).size, 40);
+    assert.equal(plan.controls.filter((control) => control.expectedOutcome === "accept")
+        .length, 10);
+    assert.deepEqual([...new Set(plan.controls.map((control) => control.kind))].sort(), [
+        "forbidden-behavior",
+        "missing-criterion",
+        "non-responsive",
+        "positive"
+    ]);
+    assert.equal(plan.controls.every((control) => control.id === `cal_${control.controlDigest}` &&
+        control.criteria === renderEvaluationCriteria(criteria) &&
+        control.prompt ===
+            renderEvaluationCandidatePrompt({
+                prompt: "Explain the required terminal event ordering.",
+                context: renderEvaluationEvidenceContext([evidenceBlock])
+            }) &&
+        (control.kind === "positive"
+            ? control.targetedCriterionIds.length === 0
+            : control.targetedCriterionIds.includes("critical"))), true);
+});
+test("control identities are order-independent while plan identity remains proposal-bound", () => {
+    const firstPayload = proposalPayload();
+    const firstCase = firstPayload.suites[0].cases[0];
+    const firstCompositionCase = firstPayload.compositionSuite.cases[0];
+    const secondPayload = {
+        ...firstPayload,
+        suites: [
+            {
+                ...firstPayload.suites[0],
+                cases: [
+                    {
+                        ...firstCase,
+                        positiveControls: [...firstCase.positiveControls].reverse(),
+                        negativeControls: [...firstCase.negativeControls].reverse()
+                    },
+                    ...firstPayload.suites[0].cases.slice(1)
+                ]
+            }
+        ],
+        compositionSuite: {
+            ...firstPayload.compositionSuite,
+            cases: [
+                {
+                    ...firstCompositionCase,
+                    negativeControls: [
+                        ...firstCompositionCase.negativeControls
+                    ].reverse()
+                },
+                ...firstPayload.compositionSuite.cases.slice(1)
+            ]
+        }
+    };
+    const firstPlan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposalFrom(firstPayload),
+        sourceInventory
+    });
+    const secondPlan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposalFrom(secondPayload),
+        sourceInventory
+    });
+    assert.deepEqual(secondPlan.controls, firstPlan.controls);
+    assert.notEqual(secondPlan.evaluationDigest, firstPlan.evaluationDigest);
+    assert.notEqual(secondPlan.planDigest, firstPlan.planDigest);
+});
+test("grader calibration rejects a proposal with stale identity", () => {
+    const valid = proposal();
+    assert.throws(() => compileEvaluationGraderCalibrationPlan({
+        proposal: {
+            ...valid,
+            judgeModel: "provider/other-judge"
+        },
+        sourceInventory
+    }), /digest does not match/u);
+});
+test("grader calibration compiles complete matching judge observations", () => {
+    const plan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposal(),
+        sourceInventory
+    });
+    const report = compileEvaluationGraderCalibrationReport({
+        plan,
+        startedAt: "2026-08-31T00:00:00.000Z",
+        finishedAt: "2026-08-31T00:01:00.000Z",
+        observations: observationsFor(plan).reverse()
+    });
+    assert.equal(report.passed, true);
+    assert.equal(report.callCount, 40);
+    assert.equal(report.matchedCount, 40);
+    assert.equal(report.acceptedCount, 10);
+    assert.equal(report.rejectedCount, 30);
+    assert.doesNotThrow(() => assertEvaluationGraderCalibrationPassed({ plan, report }));
+});
+test("grader calibration reports expectation violations and fails the admission assertion", () => {
+    const plan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposal(),
+        sourceInventory
+    });
+    const observations = observationsFor(plan);
+    const negativeIndex = plan.controls.findIndex((control) => control.expectedOutcome === "reject");
+    assert.notEqual(negativeIndex, -1);
+    observations[negativeIndex] = {
+        ...observations[negativeIndex],
+        pass: true,
+        score: EVAL_GRADER_MINIMUM_SCORE
+    };
+    const report = compileEvaluationGraderCalibrationReport({
+        plan,
+        startedAt: "2026-08-31T00:00:00.000Z",
+        finishedAt: "2026-08-31T00:01:00.000Z",
+        observations
+    });
+    assert.equal(report.passed, false);
+    assert.equal(report.matchedCount, report.callCount - 1);
+    assert.throws(() => assertEvaluationGraderCalibrationPassed({ plan, report }), /did not satisfy every control/u);
+});
+test("grader calibration rejects incomplete, duplicate, unknown, wrong-judge, reused-call, and invalid observations", () => {
+    const plan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposal(),
+        sourceInventory
+    });
+    const observations = observationsFor(plan);
+    const compile = (values, startedAt = "2026-08-31T00:00:00.000Z") => compileEvaluationGraderCalibrationReport({
+        plan,
+        startedAt,
+        finishedAt: "2026-08-31T00:01:00.000Z",
+        observations: values
+    });
+    assert.throws(() => compile(observations.slice(1)), /incomplete/u);
+    assert.throws(() => compile([observations[0], ...observations.slice(0, -1)]), /duplicate/u);
+    assert.throws(() => compile([
+        { ...observations[0], controlId: "cal_unknown" },
+        ...observations.slice(1)
+    ]), /unknown control/u);
+    assert.throws(() => compile([
+        { ...observations[0], judgeModel: "provider/wrong" },
+        ...observations.slice(1)
+    ]), /wrong judge/u);
+    assert.throws(() => compile([
+        {
+            ...observations[0],
+            callId: observations[1].callId
+        },
+        ...observations.slice(1)
+    ]), /reuse a judge call id/u);
+    assert.throws(() => compile([
+        { ...observations[0], score: 1.1 },
+        ...observations.slice(1)
+    ]), /between zero and one/u);
+    assert.throws(() => compile([
+        { ...observations[0], reason: " " },
+        ...observations.slice(1)
+    ]), /reason must not be empty/u);
+    assert.throws(() => compile([
+        { ...observations[0], callId: " " },
+        ...observations.slice(1)
+    ]), /call id must not be empty/u);
+    assert.throws(() => compile(observations, "2026-08-31"), /timestamps are invalid/u);
+});
+test("grader calibration rejects tampered plans, reports, and plan/report mismatches", () => {
+    const plan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposal(),
+        sourceInventory
+    });
+    assert.throws(() => compileEvaluationGraderCalibrationReport({
+        plan: {
+            ...plan,
+            controls: [
+                { ...plan.controls[0], response: "tampered" },
+                ...plan.controls.slice(1)
+            ]
+        },
+        startedAt: "2026-08-31T00:00:00.000Z",
+        finishedAt: "2026-08-31T00:01:00.000Z",
+        observations: observationsFor(plan)
+    }), /digest is stale/u);
+    const report = compileEvaluationGraderCalibrationReport({
+        plan,
+        startedAt: "2026-08-31T00:00:00.000Z",
+        finishedAt: "2026-08-31T00:01:00.000Z",
+        observations: observationsFor(plan)
+    });
+    const firstResult = report.results[0];
+    const incompleteResults = report.results.slice(1);
+    const redigest = (changed) => ({
+        ...changed,
+        reportDigest: evaluationGraderCalibrationReportDigest(changed)
+    });
+    assert.throws(() => assertEvaluationGraderCalibrationPassed({
+        plan,
+        report: {
+            ...report,
+            acceptedCount: report.acceptedCount + 1
+        }
+    }), /digest is stale/u);
+    assert.throws(() => assertEvaluationGraderCalibrationPassed({
+        plan,
+        report: redigest({
+            ...report,
+            callCount: incompleteResults.length,
+            acceptedCount: incompleteResults.filter((result) => result.observedOutcome === "accept").length,
+            rejectedCount: incompleteResults.filter((result) => result.observedOutcome === "reject").length,
+            matchedCount: incompleteResults.filter((result) => result.matchedExpectation).length,
+            passed: incompleteResults.every((result) => result.matchedExpectation),
+            results: incompleteResults
+        })
+    }), /summary is inconsistent/u);
+    assert.throws(() => assertEvaluationGraderCalibrationPassed({
+        plan,
+        report: redigest({
+            ...report,
+            results: [
+                {
+                    ...firstResult,
+                    callId: " "
+                },
+                ...report.results.slice(1)
+            ]
+        })
+    }), /results are inconsistent/u);
+    const otherPayload = proposalPayload();
+    const otherPlan = compileEvaluationGraderCalibrationPlan({
+        proposal: proposalFrom({
+            ...otherPayload,
+            basisDigest: "other-basis"
+        }),
+        sourceInventory
+    });
+    assert.throws(() => assertEvaluationGraderCalibrationPassed({
+        plan: otherPlan,
+        report
+    }), /does not match its expected plan/u);
+});

package/dist/test/evaluation-proposal-digest.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/evaluation-proposal-digest.test.js

@@ -0,0 +1,187 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { compileEvaluationEvidence } from "../evaluation-evidence.js";
+import { evaluationProposalDigest } from "../project-artifacts.js";
+const historicalProposal = {
+    version: 1,
+    basisDigest: "basis",
+    candidateModels: ["provider/candidate-a", "provider/candidate-b"],
+    judgeModel: "provider/judge",
+    suites: [
+        {
+            version: 1,
+            dimensionId: "protocol",
+            maximumOutputTokens: 256,
+            cases: [
+                {
+                    id: "historical-case",
+                    prompt: "Explain the protocol behavior.",
+                    rubric: "The response explains the protocol."
+                }
+            ]
+        }
+    ],
+    decompositionBenchmark: {
+        maximumVectorL1Error: 0.3,
+        cases: Array.from({ length: 5 }, (_, index) => ({
+            id: `decomposition-${String(index + 1)}`,
+            request: `Classify request ${String(index + 1)}`,
+            expected: {
+                weights: [{ dimensionId: "protocol", weight: 1 }],
+                unknownWeight: 0
+            }
+        }))
+    },
+    compositionSuite: {
+        maximumOutputTokens: 256,
+        minimumWinnerScoreGap: 0.05,
+        minimumWinnerAgreement: 0.8,
+        cases: Array.from({ length: 5 }, (_, index) => ({
+            id: `composition-${String(index + 1)}`,
+            prompt: `Compose response ${String(index + 1)}`,
+            rubric: "The response is correct.",
+            decomposition: {
+                weights: [{ dimensionId: "protocol", weight: 1 }],
+                unknownWeight: 0
+            },
+            requirements: {
+                endpoint: "responses",
+                requiresTools: false,
+                requiresVision: false
+            }
+        }))
+    }
+};
+const compiledProposal = () => {
+    const evidenceSources = compileEvaluationEvidence({
+        sources: [
+            {
+                path: "src/protocol.ts",
+                content: "The terminal event must be last.\n"
+            }
+        ],
+        maximumBlockBytes: 3_000
+    });
+    const block = evidenceSources[0].blocks[0];
+    const criteria = [
+        {
+            id: "critical",
+            description: "State that the terminal event must be last.",
+            importance: "critical",
+            gradingMode: "semantic"
+        }
+    ];
+    const compiledFields = {
+        context: `Evidence ${block.id} — src/protocol.ts (lines 1-1)\n-----\n${block.text}\n-----`,
+        rubric: "Critical criterion critical: State that the terminal event must be last.",
+        evidenceIds: [block.id],
+        criteria,
+        positiveControls: [
+            { id: "positive", response: "The terminal event must be last." }
+        ],
+        negativeControls: [
+            {
+                id: "missing",
+                response: "The response omits terminal ordering.",
+                targetedCriterionIds: ["critical"],
+                kind: "missing-criterion"
+            },
+            {
+                id: "forbidden",
+                response: "Output may follow the terminal event.",
+                targetedCriterionIds: ["critical"],
+                kind: "forbidden-behavior"
+            },
+            {
+                id: "non-responsive",
+                response: "The protocol is useful.",
+                targetedCriterionIds: ["critical"],
+                kind: "non-responsive"
+            }
+        ]
+    };
+    return {
+        ...historicalProposal,
+        evidenceSources,
+        suites: historicalProposal.suites.map((suite) => ({
+            ...suite,
+            cases: suite.cases.map((testCase) => ({
+                ...testCase,
+                ...compiledFields
+            }))
+        })),
+        compositionSuite: {
+            ...historicalProposal.compositionSuite,
+            cases: historicalProposal.compositionSuite.cases.map((testCase) => ({
+                ...testCase,
+                ...compiledFields
+            }))
+        }
+    };
+};
+test("historical proposal digest remains unchanged when compiled fields are absent", () => {
+    assert.equal(evaluationProposalDigest(historicalProposal), "cd056741905f54998a4dd16ab6952d02545f59be05b725bc4c26c904ded80720");
+});
+test("compiled proposal digest binds source bytes, evidence selections, and controls", () => {
+    const proposal = compiledProposal();
+    const digest = evaluationProposalDigest(proposal);
+    const source = proposal.evidenceSources[0];
+    const block = source.blocks[0];
+    const firstCase = proposal.suites[0].cases[0];
+    assert.notEqual(evaluationProposalDigest({
+        ...proposal,
+        evidenceSources: [
+            {
+                ...source,
+                sourceDigest: source.sourceDigest.replace(/^./u, "0"),
+                blocks: [block]
+            }
+        ]
+    }), digest);
+    assert.notEqual(evaluationProposalDigest({
+        ...proposal,
+        evidenceSources: [
+            {
+                ...source,
+                blocks: [{ ...block, text: `${block.text}changed` }]
+            }
+        ]
+    }), digest);
+    assert.notEqual(evaluationProposalDigest({
+        ...proposal,
+        suites: [
+            {
+                ...proposal.suites[0],
+                cases: [{ ...firstCase, evidenceIds: ["ev_replacement"] }]
+            }
+        ]
+    }), digest);
+    assert.notEqual(evaluationProposalDigest({
+        ...proposal,
+        suites: [
+            {
+                ...proposal.suites[0],
+                cases: [
+                    {
+                        ...firstCase,
+                        positiveControls: [
+                            { id: "positive", response: "Changed known-good response." }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }), digest);
+});
+test("proposal digest preserves the existing property-order-sensitive contract", () => {
+    const reordered = {
+        version: historicalProposal.version,
+        candidateModels: historicalProposal.candidateModels,
+        basisDigest: historicalProposal.basisDigest,
+        judgeModel: historicalProposal.judgeModel,
+        suites: historicalProposal.suites,
+        decompositionBenchmark: historicalProposal.decompositionBenchmark,
+        compositionSuite: historicalProposal.compositionSuite
+    };
+    assert.notEqual(evaluationProposalDigest(reordered), evaluationProposalDigest(historicalProposal));
+});

package/dist/test/evaluation-source-retrieval.test.d.ts

@@ -0,0 +1 @@
+export {};