@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/evaluation-evidence.js

@@ -0,0 +1,319 @@
+import { createHash } from "node:crypto";
+import { posix } from "node:path";
+const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+const bytes = (value) => Buffer.byteLength(value, "utf8");
+const evidenceId = (input) => `ev_${sha256([
+    input.path,
+    input.sourceDigest,
+    String(input.ordinal),
+    sha256(input.text)
+].join("\u0000")).slice(0, 24)}`;
+const splitOversizedText = (text, maximumBytes) => {
+    const fragments = [];
+    let fragment = "";
+    let fragmentBytes = 0;
+    for (const character of text) {
+        const characterBytes = bytes(character);
+        if (fragmentBytes + characterBytes > maximumBytes && fragment.length > 0) {
+            fragments.push(fragment);
+            fragment = "";
+            fragmentBytes = 0;
+        }
+        fragment += character;
+        fragmentBytes += characterBytes;
+    }
+    if (fragment.length > 0)
+        fragments.push(fragment);
+    return fragments;
+};
+const lineFragments = (content, maximumBytes) => {
+    const lines = content.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
+    return lines.flatMap((line, index) => {
+        const lineNumber = index + 1;
+        const fragments = bytes(line) <= maximumBytes
+            ? [line]
+            : splitOversizedText(line, maximumBytes);
+        return fragments.map((text, fragmentIndex) => ({
+            text,
+            line: lineNumber,
+            blankBoundary: fragmentIndex === fragments.length - 1 &&
+                line.replace(/\r?\n$/u, "").trim().length === 0
+        }));
+    });
+};
+const blockFragments = (fragments, maximumBytes) => {
+    const blocks = [];
+    let pending = [...fragments];
+    while (pending.length > 0) {
+        let admittedBytes = 0;
+        let admittedCount = 0;
+        let lastBlankBoundary = 0;
+        while (admittedCount < pending.length) {
+            const next = pending[admittedCount];
+            const nextBytes = bytes(next.text);
+            if (admittedBytes + nextBytes > maximumBytes)
+                break;
+            admittedBytes += nextBytes;
+            admittedCount += 1;
+            if (next.blankBoundary)
+                lastBlankBoundary = admittedCount;
+        }
+        if (admittedCount === 0) {
+            throw new Error("evidence fragment exceeds the configured byte bound");
+        }
+        const boundary = admittedCount < pending.length && lastBlankBoundary > 0
+            ? lastBlankBoundary
+            : admittedCount;
+        blocks.push(pending.slice(0, boundary));
+        pending = pending.slice(boundary);
+    }
+    return blocks;
+};
+const assertSourcePath = (path) => {
+    const segments = path.split("/");
+    if (path.length === 0 ||
+        path.includes("\u0000") ||
+        path.includes("\\") ||
+        posix.isAbsolute(path) ||
+        posix.normalize(path) !== path ||
+        segments.includes(".") ||
+        segments.includes("..") ||
+        segments.some((segment) => segment.length === 0)) {
+        throw new Error(`evidence source path is not canonical: ${JSON.stringify(path)}`);
+    }
+};
+export function compileEvaluationEvidence(input) {
+    const { maximumBlockBytes } = input;
+    if (!Number.isSafeInteger(maximumBlockBytes) || maximumBlockBytes < 4) {
+        throw new Error("evidence block byte bound must be an integer of at least four");
+    }
+    const ordered = [...input.sources].sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
+    const paths = new Set();
+    return ordered.map((source) => {
+        assertSourcePath(source.path);
+        if (paths.has(source.path)) {
+            throw new Error(`duplicate evidence source path ${JSON.stringify(source.path)}`);
+        }
+        paths.add(source.path);
+        if (source.content.length === 0) {
+            throw new Error(`evidence source is empty: ${JSON.stringify(source.path)}`);
+        }
+        const sourceDigest = sha256(source.content);
+        const blocks = blockFragments(lineFragments(source.content, maximumBlockBytes), maximumBlockBytes).map((fragments, ordinal) => {
+            const first = fragments[0];
+            const last = fragments.at(-1);
+            const text = fragments.map((fragment) => fragment.text).join("");
+            return {
+                id: evidenceId({
+                    path: source.path,
+                    sourceDigest,
+                    ordinal,
+                    text
+                }),
+                path: source.path,
+                sourceDigest,
+                contentDigest: sha256(text),
+                ordinal,
+                startLine: first.line,
+                endLine: last.line,
+                text
+            };
+        });
+        return {
+            path: source.path,
+            sourceDigest,
+            sourceBytes: bytes(source.content),
+            sourceBlockCount: blocks.length,
+            blocks
+        };
+    });
+}
+const endingLine = (startLine, text) => {
+    const newlineCount = text.split("\n").length - 1;
+    return startLine + newlineCount - (text.endsWith("\n") ? 1 : 0);
+};
+export function assertEvaluationEvidenceSources(sources) {
+    const paths = new Set();
+    let previousPath;
+    for (const source of sources) {
+        assertSourcePath(source.path);
+        if (previousPath !== undefined && previousPath >= source.path) {
+            throw new Error("compiled evidence sources are not in canonical path order");
+        }
+        previousPath = source.path;
+        if (paths.has(source.path)) {
+            throw new Error(`duplicate evidence source path ${JSON.stringify(source.path)}`);
+        }
+        paths.add(source.path);
+        if (source.blocks.length === 0) {
+            throw new Error(`compiled evidence source has no blocks: ${JSON.stringify(source.path)}`);
+        }
+        if ((source.sourceBytes === undefined) !==
+            (source.sourceBlockCount === undefined)) {
+            throw new Error(`compiled evidence source has incomplete source metadata: ${JSON.stringify(source.path)}`);
+        }
+        if (source.sourceBytes !== undefined &&
+            (!Number.isSafeInteger(source.sourceBytes) || source.sourceBytes < 1)) {
+            throw new Error(`compiled evidence source byte metadata is invalid: ${JSON.stringify(source.path)}`);
+        }
+        if (source.sourceBlockCount !== undefined &&
+            (!Number.isSafeInteger(source.sourceBlockCount) ||
+                source.sourceBlockCount < source.blocks.length)) {
+            throw new Error(`compiled evidence source block metadata is invalid: ${JSON.stringify(source.path)}`);
+        }
+        const fullCoverage = source.sourceBlockCount === undefined ||
+            (source.sourceBlockCount === source.blocks.length &&
+                source.blocks.every((block, index) => block.ordinal === index));
+        if (fullCoverage) {
+            const reconstructed = source.blocks.map((block) => block.text).join("");
+            if (sha256(reconstructed) !== source.sourceDigest ||
+                (source.sourceBytes !== undefined &&
+                    bytes(reconstructed) !== source.sourceBytes)) {
+                throw new Error(`compiled evidence source digest does not match ${JSON.stringify(source.path)}`);
+            }
+        }
+        let currentLine = 1;
+        let previousOrdinal = -1;
+        let previousStartLine = 0;
+        for (const [index, block] of source.blocks.entries()) {
+            if (block.path !== source.path ||
+                block.sourceDigest !== source.sourceDigest ||
+                block.contentDigest !== sha256(block.text) ||
+                block.id !==
+                    evidenceId({
+                        path: source.path,
+                        sourceDigest: source.sourceDigest,
+                        ordinal: block.ordinal,
+                        text: block.text
+                    })) {
+                throw new Error(`compiled evidence block identity does not match ${JSON.stringify(block.id)}`);
+            }
+            const expectedStartLine = fullCoverage ? currentLine : block.startLine;
+            const expectedEndLine = endingLine(expectedStartLine, block.text);
+            if (block.startLine !== expectedStartLine ||
+                block.endLine !== expectedEndLine ||
+                block.ordinal <= previousOrdinal ||
+                block.startLine < previousStartLine ||
+                block.startLine < 1 ||
+                block.endLine < block.startLine ||
+                (source.sourceBlockCount !== undefined &&
+                    block.ordinal >= source.sourceBlockCount)) {
+                throw new Error(`compiled evidence block line metadata does not match ${JSON.stringify(block.id)}`);
+            }
+            if (fullCoverage && block.ordinal !== index) {
+                throw new Error(`compiled evidence block ordinal does not match ${JSON.stringify(block.id)}`);
+            }
+            currentLine =
+                expectedEndLine + (block.text.endsWith("\n") ? 1 : 0);
+            previousOrdinal = block.ordinal;
+            previousStartLine = block.startLine;
+        }
+    }
+}
+export function selectEvaluationEvidenceBlocks(input) {
+    assertEvaluationEvidenceSources(input.sources);
+    const selectedIds = new Set(input.evidenceIds);
+    if (selectedIds.size !== input.evidenceIds.length) {
+        throw new Error("compiled evidence block selection contains duplicate ids");
+    }
+    const selected = input.sources
+        .map((source) => ({
+        ...source,
+        blocks: source.blocks.filter((block) => selectedIds.delete(block.id))
+    }))
+        .filter((source) => source.blocks.length > 0);
+    if (selectedIds.size > 0) {
+        throw new Error(`compiled evidence block selection contains unknown ids: ${[
+            ...selectedIds
+        ].join(", ")}`);
+    }
+    assertEvaluationEvidenceSources(selected);
+    return selected;
+}
+export function mergeEvaluationEvidenceSources(packets) {
+    const byPath = new Map();
+    for (const packet of packets) {
+        assertEvaluationEvidenceSources(packet);
+        for (const source of packet) {
+            const existing = byPath.get(source.path);
+            if (existing === undefined) {
+                byPath.set(source.path, source);
+                continue;
+            }
+            if (existing.sourceDigest !== source.sourceDigest ||
+                existing.sourceBytes !== source.sourceBytes ||
+                existing.sourceBlockCount !== source.sourceBlockCount) {
+                throw new Error(`evidence source changed between authoring packets: ${JSON.stringify(source.path)}`);
+            }
+            const byOrdinal = new Map(existing.blocks.map((block) => [block.ordinal, block]));
+            for (const block of source.blocks) {
+                const current = byOrdinal.get(block.ordinal);
+                if (current !== undefined &&
+                    JSON.stringify(current) !== JSON.stringify(block)) {
+                    throw new Error(`evidence source block changed between authoring packets: ${JSON.stringify(block.id)}`);
+                }
+                byOrdinal.set(block.ordinal, block);
+            }
+            byPath.set(source.path, {
+                ...existing,
+                blocks: [...byOrdinal.values()].sort((left, right) => left.ordinal - right.ordinal)
+            });
+        }
+    }
+    const merged = [...byPath.values()].sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
+    assertEvaluationEvidenceSources(merged);
+    return merged;
+}
+const evidenceBlocksById = (sources) => {
+    assertEvaluationEvidenceSources(sources);
+    const byId = new Map();
+    for (const source of sources) {
+        for (const block of source.blocks) {
+            if (byId.has(block.id)) {
+                throw new Error(`duplicate compiled evidence id ${JSON.stringify(block.id)}`);
+            }
+            byId.set(block.id, block);
+        }
+    }
+    return byId;
+};
+export function resolveEvaluationEvidence(input) {
+    if (input.evidenceIds.length === 0) {
+        throw new Error("evaluation case must select at least one evidence block");
+    }
+    const { maximumContextBytes } = input;
+    if (!Number.isSafeInteger(maximumContextBytes) || maximumContextBytes < 1) {
+        throw new Error("evidence context byte bound must be a positive integer");
+    }
+    const byId = evidenceBlocksById(input.sources);
+    const selected = new Set();
+    let selectedBytes = 0;
+    return input.evidenceIds.map((id) => {
+        if (selected.has(id)) {
+            throw new Error(`evaluation case selects duplicate evidence id ${JSON.stringify(id)}`);
+        }
+        selected.add(id);
+        const block = byId.get(id);
+        if (block === undefined) {
+            throw new Error(`evaluation case selects unknown evidence id ${JSON.stringify(id)}`);
+        }
+        selectedBytes += bytes(block.text);
+        if (selectedBytes > maximumContextBytes) {
+            throw new Error(`evaluation case evidence exceeds the ${String(maximumContextBytes)} byte context bound`);
+        }
+        return block;
+    });
+}
+export function renderEvaluationEvidenceContext(blocks) {
+    if (blocks.length === 0) {
+        throw new Error("evaluation context requires at least one resolved evidence block");
+    }
+    return blocks
+        .map((block) => [
+        `Evidence ${block.id} — ${block.path} (lines ${String(block.startLine)}-${String(block.endLine)})`,
+        "-----",
+        block.text,
+        "-----"
+    ].join("\n"))
+        .join("\n\n");
+}

package/dist/evaluation-grader-calibration-protocol.d.ts

@@ -0,0 +1,108 @@
+import { Schema } from "effect";
+export declare const EVAL_GRADER_CALIBRATION_POLICY_VERSION: 1;
+export declare const EvalGraderCalibrationControl: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly controlDigest: Schema.String;
+    readonly suiteKind: Schema.Literals<readonly ["dimension", "composition"]>;
+    readonly suiteId: Schema.String;
+    readonly caseId: Schema.String;
+    readonly authoredControlId: Schema.String;
+    readonly kind: Schema.Literals<readonly ["positive", "missing-criterion", "forbidden-behavior", "non-responsive"]>;
+    readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+    readonly expectedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+    readonly prompt: Schema.String;
+    readonly criteria: Schema.String;
+    readonly response: Schema.String;
+}>;
+export type EvalGraderCalibrationControl = typeof EvalGraderCalibrationControl.Type;
+export declare const EvalGraderCalibrationPlan: Schema.Struct<{
+    readonly version: Schema.Literal<1>;
+    readonly policyVersion: Schema.Literal<1>;
+    readonly evaluationDigest: Schema.String;
+    readonly judgeProtocolVersion: Schema.Literal<1>;
+    readonly judgeModel: Schema.String;
+    readonly judgeReasoningEffort: Schema.Null;
+    readonly judgeSystemPrompt: Schema.String;
+    readonly minimumJudgeScore: Schema.Finite;
+    readonly callCount: Schema.Finite;
+    readonly controls: Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly controlDigest: Schema.String;
+        readonly suiteKind: Schema.Literals<readonly ["dimension", "composition"]>;
+        readonly suiteId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly authoredControlId: Schema.String;
+        readonly kind: Schema.Literals<readonly ["positive", "missing-criterion", "forbidden-behavior", "non-responsive"]>;
+        readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+        readonly expectedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+        readonly prompt: Schema.String;
+        readonly criteria: Schema.String;
+        readonly response: Schema.String;
+    }>>;
+    readonly planDigest: Schema.String;
+}>;
+export type EvalGraderCalibrationPlan = typeof EvalGraderCalibrationPlan.Type;
+export declare const EvalGraderCalibrationObservation: Schema.Struct<{
+    readonly controlId: Schema.String;
+    readonly judgeModel: Schema.String;
+    readonly callId: Schema.String;
+    readonly pass: Schema.Boolean;
+    readonly score: Schema.Finite;
+    readonly reason: Schema.String;
+    readonly costUsd: Schema.optionalKey<Schema.Finite>;
+    readonly durationMs: Schema.optionalKey<Schema.Finite>;
+    readonly inputTokens: Schema.optionalKey<Schema.Finite>;
+    readonly outputTokens: Schema.optionalKey<Schema.Finite>;
+}>;
+export type EvalGraderCalibrationObservation = typeof EvalGraderCalibrationObservation.Type;
+export declare const EvalGraderCalibrationResult: Schema.Struct<{
+    readonly expectedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+    readonly observedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+    readonly matchedExpectation: Schema.Boolean;
+    readonly controlId: Schema.String;
+    readonly judgeModel: Schema.String;
+    readonly callId: Schema.String;
+    readonly pass: Schema.Boolean;
+    readonly score: Schema.Finite;
+    readonly reason: Schema.String;
+    readonly costUsd: Schema.optionalKey<Schema.Finite>;
+    readonly durationMs: Schema.optionalKey<Schema.Finite>;
+    readonly inputTokens: Schema.optionalKey<Schema.Finite>;
+    readonly outputTokens: Schema.optionalKey<Schema.Finite>;
+}>;
+export type EvalGraderCalibrationResult = typeof EvalGraderCalibrationResult.Type;
+export declare const EvalGraderCalibrationReport: Schema.Struct<{
+    readonly version: Schema.Literal<1>;
+    readonly policyVersion: Schema.Literal<1>;
+    readonly planDigest: Schema.String;
+    readonly evaluationDigest: Schema.String;
+    readonly judgeProtocolVersion: Schema.Literal<1>;
+    readonly judgeModel: Schema.String;
+    readonly judgeReasoningEffort: Schema.Null;
+    readonly judgeSystemPrompt: Schema.String;
+    readonly minimumJudgeScore: Schema.Finite;
+    readonly startedAt: Schema.String;
+    readonly finishedAt: Schema.String;
+    readonly callCount: Schema.Finite;
+    readonly acceptedCount: Schema.Finite;
+    readonly rejectedCount: Schema.Finite;
+    readonly matchedCount: Schema.Finite;
+    readonly passed: Schema.Boolean;
+    readonly results: Schema.$Array<Schema.Struct<{
+        readonly expectedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+        readonly observedOutcome: Schema.Literals<readonly ["accept", "reject"]>;
+        readonly matchedExpectation: Schema.Boolean;
+        readonly controlId: Schema.String;
+        readonly judgeModel: Schema.String;
+        readonly callId: Schema.String;
+        readonly pass: Schema.Boolean;
+        readonly score: Schema.Finite;
+        readonly reason: Schema.String;
+        readonly costUsd: Schema.optionalKey<Schema.Finite>;
+        readonly durationMs: Schema.optionalKey<Schema.Finite>;
+        readonly inputTokens: Schema.optionalKey<Schema.Finite>;
+        readonly outputTokens: Schema.optionalKey<Schema.Finite>;
+    }>>;
+    readonly reportDigest: Schema.String;
+}>;
+export type EvalGraderCalibrationReport = typeof EvalGraderCalibrationReport.Type;

package/dist/evaluation-grader-calibration-protocol.js

@@ -0,0 +1,80 @@
+import { Schema } from "effect";
+import { EVAL_GRADER_JUDGE_PROTOCOL_VERSION } from "./evaluation-grading-policy.js";
+export const EVAL_GRADER_CALIBRATION_POLICY_VERSION = 1;
+const NonNegativeInteger = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && Number.isInteger(value)
+    ? undefined
+    : "value must be a non-negative integer")));
+const UnitInterval = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && value <= 1
+    ? undefined
+    : "value must be between zero and one")));
+const NonNegativeFinite = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 ? undefined : "value must be non-negative")));
+export const EvalGraderCalibrationControl = Schema.Struct({
+    id: Schema.String,
+    controlDigest: Schema.String,
+    suiteKind: Schema.Literals(["dimension", "composition"]),
+    suiteId: Schema.String,
+    caseId: Schema.String,
+    authoredControlId: Schema.String,
+    kind: Schema.Literals([
+        "positive",
+        "missing-criterion",
+        "forbidden-behavior",
+        "non-responsive"
+    ]),
+    targetedCriterionIds: Schema.Array(Schema.String),
+    expectedOutcome: Schema.Literals(["accept", "reject"]),
+    prompt: Schema.String,
+    criteria: Schema.String,
+    response: Schema.String
+});
+export const EvalGraderCalibrationPlan = Schema.Struct({
+    version: Schema.Literal(1),
+    policyVersion: Schema.Literal(EVAL_GRADER_CALIBRATION_POLICY_VERSION),
+    evaluationDigest: Schema.String,
+    judgeProtocolVersion: Schema.Literal(EVAL_GRADER_JUDGE_PROTOCOL_VERSION),
+    judgeModel: Schema.String,
+    judgeReasoningEffort: Schema.Null,
+    judgeSystemPrompt: Schema.String,
+    minimumJudgeScore: UnitInterval,
+    callCount: NonNegativeInteger,
+    controls: Schema.Array(EvalGraderCalibrationControl),
+    planDigest: Schema.String
+});
+export const EvalGraderCalibrationObservation = Schema.Struct({
+    controlId: Schema.String,
+    judgeModel: Schema.String,
+    callId: Schema.String,
+    pass: Schema.Boolean,
+    score: UnitInterval,
+    reason: Schema.String,
+    costUsd: Schema.optionalKey(NonNegativeFinite),
+    durationMs: Schema.optionalKey(NonNegativeFinite),
+    inputTokens: Schema.optionalKey(NonNegativeInteger),
+    outputTokens: Schema.optionalKey(NonNegativeInteger)
+});
+export const EvalGraderCalibrationResult = Schema.Struct({
+    ...EvalGraderCalibrationObservation.fields,
+    expectedOutcome: Schema.Literals(["accept", "reject"]),
+    observedOutcome: Schema.Literals(["accept", "reject"]),
+    matchedExpectation: Schema.Boolean
+});
+export const EvalGraderCalibrationReport = Schema.Struct({
+    version: Schema.Literal(1),
+    policyVersion: Schema.Literal(EVAL_GRADER_CALIBRATION_POLICY_VERSION),
+    planDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    judgeProtocolVersion: Schema.Literal(EVAL_GRADER_JUDGE_PROTOCOL_VERSION),
+    judgeModel: Schema.String,
+    judgeReasoningEffort: Schema.Null,
+    judgeSystemPrompt: Schema.String,
+    minimumJudgeScore: UnitInterval,
+    startedAt: Schema.String,
+    finishedAt: Schema.String,
+    callCount: NonNegativeInteger,
+    acceptedCount: NonNegativeInteger,
+    rejectedCount: NonNegativeInteger,
+    matchedCount: NonNegativeInteger,
+    passed: Schema.Boolean,
+    results: Schema.Array(EvalGraderCalibrationResult),
+    reportDigest: Schema.String
+});

package/dist/evaluation-grader-calibration.d.ts

@@ -0,0 +1,18 @@
+import { type EvalGraderCalibrationObservation as CalibrationObservation, type EvalGraderCalibrationPlan as CalibrationPlan, type EvalGraderCalibrationReport as CalibrationReport } from "./evaluation-grader-calibration-protocol.js";
+import type { EvalEvaluationProposal } from "./project-contracts.js";
+export { EVAL_GRADER_CALIBRATION_POLICY_VERSION, EvalGraderCalibrationControl, EvalGraderCalibrationObservation, EvalGraderCalibrationPlan, EvalGraderCalibrationReport, EvalGraderCalibrationResult } from "./evaluation-grader-calibration-protocol.js";
+export declare const evaluationGraderCalibrationReportDigest: (report: Omit<CalibrationReport, "reportDigest">) => string;
+export declare function compileEvaluationGraderCalibrationPlan(input: {
+    readonly proposal: EvalEvaluationProposal;
+    readonly sourceInventory: readonly string[];
+}): CalibrationPlan;
+export declare function compileEvaluationGraderCalibrationReport(input: {
+    readonly plan: CalibrationPlan;
+    readonly startedAt: string;
+    readonly finishedAt: string;
+    readonly observations: readonly CalibrationObservation[];
+}): CalibrationReport;
+export declare function assertEvaluationGraderCalibrationPassed(input: {
+    readonly plan: CalibrationPlan;
+    readonly report: CalibrationReport;
+}): void;