@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/eval-event-log.js

@@ -0,0 +1,220 @@
+import { Schema } from "effect";
+const NonNegativeInteger = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && Number.isInteger(value) ? undefined : "value must be a non-negative integer")));
+const PositiveInteger = NonNegativeInteger.pipe(Schema.check(Schema.makeFilter((value) => value >= 1 ? undefined : "value must be a positive integer")));
+const NonNegativeFinite = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 ? undefined : "value must be a non-negative finite number")));
+const BoundedFailureReason = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => value.length >= 1 && value.length <= 2_000
+    ? undefined
+    : "failure reason must contain between 1 and 2000 characters")));
+const RunPlanEvent = Schema.Struct({
+    name: Schema.Literal("plan"),
+    at: Schema.String,
+    runId: Schema.String,
+    plannedCalls: NonNegativeInteger
+});
+const RunCallEventFields = {
+    at: Schema.String,
+    runId: Schema.String,
+    dimensionId: Schema.String,
+    caseId: Schema.String,
+    sequence: PositiveInteger
+};
+const RunCandidateEvent = Schema.Struct({
+    name: Schema.Literal("candidate#"),
+    ...RunCallEventFields
+});
+const RunJudgeEvent = Schema.Struct({
+    name: Schema.Literal("judge#"),
+    ...RunCallEventFields
+});
+export const EvalRunFailurePhase = Schema.Literals([
+    "target",
+    "classifier",
+    "suite-inspection",
+    "comparison",
+    "activation",
+    "accounting",
+    "completion"
+]);
+const RunFailureDetailEvent = Schema.Struct({
+    name: Schema.Literal("failure-detail"),
+    at: Schema.String,
+    runId: Schema.String,
+    phase: EvalRunFailurePhase,
+    reason: BoundedFailureReason,
+    retryable: Schema.Boolean,
+    callId: Schema.optionalKey(Schema.String),
+    reservationId: Schema.optionalKey(Schema.String),
+    accounting: Schema.Struct({
+        expectedCalls: NonNegativeInteger,
+        observedCalls: NonNegativeInteger,
+        knownInputTokens: NonNegativeInteger,
+        knownOutputTokens: NonNegativeInteger,
+        unknownTokenMeasurements: NonNegativeInteger,
+        knownPricedSubtotalUsd: NonNegativeFinite,
+        unpricedCalls: NonNegativeInteger,
+        subscriptionUnitCalls: NonNegativeInteger
+    }),
+    cleanup: Schema.Struct({
+        sessionOpened: Schema.Boolean,
+        sessionClosed: Schema.Boolean,
+        detail: Schema.optionalKey(Schema.String)
+    })
+});
+const AuthoringEventFields = {
+    at: Schema.String,
+    operationId: Schema.String
+};
+const AuthoringBasisEvent = Schema.Struct({
+    name: Schema.Literal("basis"),
+    ...AuthoringEventFields
+});
+const AuthoringDimensionsEvent = Schema.Struct({
+    name: Schema.Literal("dimensions"),
+    ...AuthoringEventFields
+});
+const AuthoringActivationEvent = Schema.Struct({
+    name: Schema.Literal("activation"),
+    ...AuthoringEventFields
+});
+export const EvalNamedEvent = Schema.Union([
+    RunPlanEvent,
+    RunCandidateEvent,
+    RunJudgeEvent,
+    RunFailureDetailEvent,
+    AuthoringBasisEvent,
+    AuthoringDimensionsEvent,
+    AuthoringActivationEvent
+]);
+export const EvalNamedEventLog = Schema.Array(EvalNamedEvent);
+const caseKey = (dimensionId, caseId) => `${dimensionId}\u0000${caseId}`;
+export function resumeEvalCaseCursors(log) {
+    const cursors = new Map();
+    for (const event of log) {
+        if (event.name !== "candidate#" && event.name !== "judge#")
+            continue;
+        const key = caseKey(event.dimensionId, event.caseId);
+        const cursor = cursors.get(key) ??
+            {
+                dimensionId: event.dimensionId,
+                caseId: event.caseId,
+                candidates: new Map(),
+                judges: new Set()
+            };
+        const firstHoleSequence = (() => {
+            let sequence = 1;
+            while (cursor.candidates.has(sequence) && cursor.judges.has(sequence)) {
+                sequence += 1;
+            }
+            return sequence;
+        })();
+        if (cursor.candidates.has(firstHoleSequence) &&
+            !cursor.judges.has(firstHoleSequence) &&
+            (event.name !== "judge#" || event.sequence !== firstHoleSequence)) {
+            cursors.set(key, cursor);
+            continue;
+        }
+        if (event.name === "candidate#")
+            cursor.candidates.set(event.sequence, event);
+        else
+            cursor.judges.add(event.sequence);
+        cursor.lastEvent = event;
+        cursors.set(key, cursor);
+    }
+    return [...cursors.values()].map((cursor) => {
+        let completePairs = 0;
+        while (cursor.candidates.has(completePairs + 1) &&
+            cursor.judges.has(completePairs + 1)) {
+            completePairs += 1;
+        }
+        const hole = cursor.candidates.get(completePairs + 1);
+        return {
+            dimensionId: cursor.dimensionId,
+            caseId: cursor.caseId,
+            completePairs,
+            ...(hole === undefined
+                ? cursor.lastEvent === undefined
+                    ? {}
+                    : { lastEvent: cursor.lastEvent }
+                : { lastEvent: hole }),
+            ...(hole === undefined ? {} : { hole })
+        };
+    });
+}
+export function evalEventLabel(event) {
+    return event.name === "candidate#" || event.name === "judge#"
+        ? `${event.name}${String(event.sequence)}`
+        : event.name;
+}
+const lastPlanEvent = (log) => {
+    for (let index = log.length - 1; index >= 0; index -= 1) {
+        const event = log[index];
+        if (event?.name === "plan")
+            return event;
+    }
+    return undefined;
+};
+export function presentEvalEventLog(log, options = {}) {
+    const plan = lastPlanEvent(log);
+    if (plan !== undefined && plan.plannedCalls < 1) {
+        return {
+            failed: true,
+            lines: ["run fail no planned calls"]
+        };
+    }
+    const cursors = resumeEvalCaseCursors(log);
+    const failHoles = options.failHoles ?? true;
+    const lastEvent = log[log.length - 1];
+    const failureLine = lastEvent?.name === "failure-detail"
+        ? `run fail ${lastEvent.phase}: ${lastEvent.reason}`
+        : undefined;
+    const authoringLine = lastEvent?.name === "basis" ||
+        lastEvent?.name === "dimensions" ||
+        lastEvent?.name === "activation"
+        ? `authoring ${evalEventLabel(lastEvent)}`
+        : undefined;
+    const cursorLines = cursors.map((cursor) => {
+        const label = cursor.lastEvent === undefined ? "plan" : evalEventLabel(cursor.lastEvent);
+        return `${cursor.dimensionId}/${cursor.caseId} ${cursor.hole === undefined || !failHoles
+            ? label
+            : `fail ${label} missing judge#${String(cursor.hole.sequence)}`}`;
+    });
+    return {
+        failed: failureLine !== undefined ||
+            (failHoles && cursors.some((cursor) => cursor.hole !== undefined)),
+        lines: failureLine !== undefined
+            ? [...cursorLines, failureLine]
+            : cursors.length === 0
+                ? authoringLine === undefined
+                    ? plan === undefined
+                        ? []
+                        : [`run ${evalEventLabel(plan)}`]
+                    : [authoringLine]
+                : authoringLine === undefined
+                    ? cursorLines
+                    : [...cursorLines, authoringLine]
+    };
+}
+export function assertEvalRunCanFail(log) {
+    const plan = lastPlanEvent(log);
+    if (plan === undefined) {
+        throw new Error("failed eval run cannot terminate without its plan event");
+    }
+    const terminal = log[log.length - 1];
+    if (terminal?.name !== "failure-detail" || terminal.runId !== plan.runId) {
+        throw new Error("failed eval run must end with a matching terminal failure-detail event");
+    }
+}
+export function assertEvalRunCanComplete(log) {
+    const plan = lastPlanEvent(log);
+    if (plan === undefined || plan.plannedCalls < 1) {
+        throw new Error("eval run cannot complete without a plan containing at least one call");
+    }
+    const failure = log.find((event) => event.name === "failure-detail" && event.runId === plan.runId);
+    if (failure !== undefined) {
+        throw new Error(`eval run cannot complete after terminal failure in phase ${JSON.stringify(failure.phase)}`);
+    }
+    const hole = resumeEvalCaseCursors(log).find((cursor) => cursor.hole !== undefined);
+    if (hole?.hole !== undefined) {
+        throw new Error(`eval case ${JSON.stringify(hole.dimensionId)}/${JSON.stringify(hole.caseId)} cannot complete after ${evalEventLabel(hole.hole)} without judge#${String(hole.hole.sequence)}`);
+    }
+}

package/dist/evaluation-authoring-policy.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Provisional transport limits for compiler-backed evaluation authoring.
+ *
+ * These values bound model input/output and persisted candidate context; they
+ * are not semantic-quality thresholds. Live preflight and later empirical
+ * calibration must determine whether they produce coherent, useful cases.
+ */
+export declare const EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES = 3000;
+export declare const EVAL_AUTHORING_CASE_EVIDENCE_BYTES = 12000;
+/**
+ * Maximum compiler blocks exposed as one mechanically safe author selection.
+ *
+ * A block may consume the complete block-byte allowance, so this cardinality
+ * guarantees that selecting every exposed block cannot exceed the case context
+ * allowance. The exact byte check remains authoritative because smaller blocks
+ * do not make every same-cardinality combination equivalent.
+ */
+export declare const EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS: number;

package/dist/evaluation-authoring-policy.js

@@ -0,0 +1,19 @@
+/**
+ * Provisional transport limits for compiler-backed evaluation authoring.
+ *
+ * These values bound model input/output and persisted candidate context; they
+ * are not semantic-quality thresholds. Live preflight and later empirical
+ * calibration must determine whether they produce coherent, useful cases.
+ */
+export const EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES = 3_000;
+export const EVAL_AUTHORING_CASE_EVIDENCE_BYTES = 12_000;
+/**
+ * Maximum compiler blocks exposed as one mechanically safe author selection.
+ *
+ * A block may consume the complete block-byte allowance, so this cardinality
+ * guarantees that selecting every exposed block cannot exceed the case context
+ * allowance. The exact byte check remains authoritative because smaller blocks
+ * do not make every same-cardinality combination equivalent.
+ */
+export const EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS = Math.floor(EVAL_AUTHORING_CASE_EVIDENCE_BYTES /
+    EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES);

package/dist/evaluation-authoring-validation.d.ts

@@ -0,0 +1,22 @@
+export type EvalAuthoringValidationCategory = "schema-constraint" | "schema-decode" | "case-shape" | "criterion-topology" | "control-topology" | "evidence-selection";
+export declare class EvalAuthoringValidationIssue extends Error {
+    readonly category: EvalAuthoringValidationCategory;
+    readonly path: string;
+    constructor(input: {
+        readonly category: EvalAuthoringValidationCategory;
+        readonly path: string;
+        readonly detail: string;
+        readonly cause?: unknown;
+    });
+}
+export declare const evalAuthoringValidationDiagnostic: (cause: unknown) => {
+    readonly category: EvalAuthoringValidationCategory;
+    readonly path: string;
+    readonly detail: string;
+};
+export declare const prefixEvalAuthoringValidationIssue: (cause: unknown, prefix: string) => EvalAuthoringValidationIssue;
+export declare const renderEvalAuthoringValidationFailure: (input: {
+    readonly scope: string;
+    readonly cause: unknown;
+    readonly callId?: string;
+}) => string;

package/dist/evaluation-authoring-validation.js

@@ -0,0 +1,72 @@
+export class EvalAuthoringValidationIssue extends Error {
+    category;
+    path;
+    constructor(input) {
+        super(input.detail, input.cause === undefined ? undefined : { cause: input.cause });
+        this.name = "EvalAuthoringValidationIssue";
+        this.category = input.category;
+        this.path = input.path;
+    }
+}
+const recordOf = (value) => typeof value === "object" && value !== null && !Array.isArray(value)
+    ? value
+    : undefined;
+const schemaIssuePath = (value) => {
+    const record = recordOf(value);
+    if (record === undefined)
+        return undefined;
+    if (Array.isArray(record.path))
+        return record.path;
+    if (Array.isArray(record.issues)) {
+        for (const issue of record.issues) {
+            const path = schemaIssuePath(issue);
+            if (path !== undefined)
+                return path;
+        }
+    }
+    return schemaIssuePath(record.issue);
+};
+const propertyPath = (segments) => segments.reduce((path, segment) => {
+    if (typeof segment === "number")
+        return `${path}[${String(segment)}]`;
+    if (typeof segment === "string" && /^[A-Za-z_$][A-Za-z0-9_$]*$/u.test(segment)) {
+        return `${path}.${segment}`;
+    }
+    return `${path}[${JSON.stringify(String(segment))}]`;
+}, "$");
+export const evalAuthoringValidationDiagnostic = (cause) => {
+    if (cause instanceof EvalAuthoringValidationIssue) {
+        return {
+            category: cause.category,
+            path: cause.path,
+            detail: cause.message
+        };
+    }
+    const record = recordOf(cause);
+    const issuePath = schemaIssuePath(record?.issue);
+    return {
+        category: "schema-decode",
+        path: issuePath === undefined ? "$" : propertyPath(issuePath),
+        detail: "value does not conform to the persisted evaluation schema"
+    };
+};
+export const prefixEvalAuthoringValidationIssue = (cause, prefix) => {
+    const diagnostic = evalAuthoringValidationDiagnostic(cause);
+    const suffix = diagnostic.path === "$" ? "" : diagnostic.path.slice(1);
+    return new EvalAuthoringValidationIssue({
+        category: diagnostic.category,
+        path: `${prefix}${suffix}`,
+        detail: diagnostic.detail,
+        cause
+    });
+};
+export const renderEvalAuthoringValidationFailure = (input) => {
+    const diagnostic = evalAuthoringValidationDiagnostic(input.cause);
+    return [
+        `${input.scope} failed validation`,
+        ...(input.callId === undefined ? [] : [`call id ${input.callId}`]),
+        `category ${diagnostic.category}`,
+        `path ${diagnostic.path}`,
+        diagnostic.detail
+    ].join("; ");
+};

package/dist/evaluation-evidence.d.ts

@@ -0,0 +1,20 @@
+import type { EvalEvidenceBlock, EvalEvidenceSource } from "./project-contracts.js";
+export declare function compileEvaluationEvidence(input: {
+    readonly sources: readonly {
+        readonly path: string;
+        readonly content: string;
+    }[];
+    readonly maximumBlockBytes: number;
+}): readonly EvalEvidenceSource[];
+export declare function assertEvaluationEvidenceSources(sources: readonly EvalEvidenceSource[]): void;
+export declare function selectEvaluationEvidenceBlocks(input: {
+    readonly sources: readonly EvalEvidenceSource[];
+    readonly evidenceIds: readonly string[];
+}): readonly EvalEvidenceSource[];
+export declare function mergeEvaluationEvidenceSources(packets: readonly (readonly EvalEvidenceSource[])[]): readonly EvalEvidenceSource[];
+export declare function resolveEvaluationEvidence(input: {
+    readonly sources: readonly EvalEvidenceSource[];
+    readonly evidenceIds: readonly string[];
+    readonly maximumContextBytes: number;
+}): readonly EvalEvidenceBlock[];
+export declare function renderEvaluationEvidenceContext(blocks: readonly EvalEvidenceBlock[]): string;