@velum-labs/routekit-eval-service 1.0.0

package/dist/production-runner.d.ts

@@ -0,0 +1,21 @@
+import { Layer } from "effect";
+import { HttpClient } from "effect/unstable/http";
+import type { RouteKitEvalServiceOptions } from "./layer-options.js";
+import { EvalService, type EvalServiceConfiguration } from "./service.js";
+export type { RouteKitEvalServiceOptions };
+declare const EvalServiceCredentialError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "EvalServiceCredentialError";
+} & Readonly<A>;
+export declare class EvalServiceCredentialError extends EvalServiceCredentialError_base<{
+    readonly detail: string;
+}> {
+    get message(): string;
+}
+/**
+ * Complete production composition for RouteKit Eval.
+ *
+ * EvalService consumes the vendored engine's native Effect service. The
+ * execution port is the only adapter; no parallel comparison-runner service
+ * mirrors or wraps the engine API.
+ */
+export declare const makeRouteKitEvalServiceLayer: (configuration: EvalServiceConfiguration, options: RouteKitEvalServiceOptions) => Layer.Layer<EvalService, never, HttpClient.HttpClient>;

package/dist/production-runner.js

@@ -0,0 +1,39 @@
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { EvalEngineExecutionError, EvalExecutionPort, makeEvalEngineLayer, makeRouteKitEvalExecutionPortService } from "@velum-labs/routekit-eval-engine";
+import { Data, Effect, Layer, Stream } from "effect";
+import { HttpClient } from "effect/unstable/http";
+import { makeEvalServiceLayer } from "./service.js";
+export class EvalServiceCredentialError extends Data.TaggedError("EvalServiceCredentialError") {
+    get message() {
+        return this.detail;
+    }
+}
+const makeProductionExecutionPort = (options) => Effect.map(HttpClient.HttpClient, (httpClient) => {
+    const bearerCredential = options.bearerCredential?.trim();
+    if (bearerCredential === undefined || bearerCredential.length === 0) {
+        return {
+            execute: () => Stream.fail(new EvalEngineExecutionError({
+                cause: new EvalServiceCredentialError({
+                    detail: "RouteKit Eval comparison execution requires an injected bearer credential."
+                }),
+                detail: "RouteKit Eval comparison execution requires an injected bearer credential."
+            }))
+        };
+    }
+    return makeRouteKitEvalExecutionPortService({
+        bearerCredential,
+        ...(options.childEnvironment === undefined
+            ? {}
+            : { childEnvironment: options.childEnvironment }),
+        ...(options.execPath === undefined ? {} : { execPath: options.execPath })
+    }, httpClient);
+});
+const makeProductionEvalEngineLayer = (options) => makeEvalEngineLayer().pipe(Layer.provide(Layer.effect(EvalExecutionPort, makeProductionExecutionPort(options))));
+/**
+ * Complete production composition for RouteKit Eval.
+ *
+ * EvalService consumes the vendored engine's native Effect service. The
+ * execution port is the only adapter; no parallel comparison-runner service
+ * mirrors or wraps the engine API.
+ */
+export const makeRouteKitEvalServiceLayer = (configuration, options) => makeEvalServiceLayer(configuration).pipe(Layer.provide(makeProductionEvalEngineLayer(options)), Layer.provide(NodeServicesLayer));

package/dist/service.d.ts

@@ -0,0 +1,56 @@
+import type { EvalComparisonRequest, EvalComparisonResult, PublishedRoutingActivation, RoutingActivationConstraints, RoutingBasis, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
+import { EvalRunManifest } from "@velum-labs/routekit-eval-contracts";
+import { EvalEngine } from "@velum-labs/routekit-eval-engine";
+import { Context, Effect, FileSystem, Layer, Path } from "effect";
+import { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
+export type EvalComparisonMode = "pilot" | "full";
+export type EvalSuiteInspection = {
+    readonly suiteDigest: string;
+    readonly manifest: EvalRunManifest;
+};
+export type EvalComparisonEstimate = {
+    readonly callCount: number;
+    readonly maximumCostUsd?: number;
+    readonly pricingKnown: boolean;
+};
+export type EvalRunConfiguration = {
+    readonly concurrency?: number;
+    readonly timeoutMs?: number;
+};
+export type EvalServiceConfiguration = {
+    readonly gatewayUrl?: string;
+    readonly snapshotRoot?: string;
+    readonly full?: EvalRunConfiguration;
+};
+export type DimensionMatrixSuite = {
+    readonly dimensionId: string;
+    readonly suitePath: string;
+};
+export type DimensionMatrixQualificationInput = {
+    readonly basis: RoutingBasis;
+    readonly candidateModels: ReadonlyArray<string>;
+    readonly classifierModel: string;
+    readonly judgeModel: string;
+    readonly objective: RoutingObjectivePolicy;
+    readonly maximumUnknownWeight: number;
+    readonly constraints?: RoutingActivationConstraints;
+    readonly suites: ReadonlyArray<DimensionMatrixSuite>;
+};
+export type DimensionMatrixQualificationResult = {
+    readonly comparisons: ReadonlyArray<EvalComparisonResult>;
+    readonly snapshot: PublishedRoutingActivation;
+};
+export type EvalServiceError = EvalServiceComparisonError | EvalServiceConfigurationError | EvalServiceEstimateError | EvalServicePolicyError | EvalServicePublicationError | EvalServiceSpendLimitError | EvalServiceValidationError;
+export type EvalServiceShape = {
+    readonly validate: (suitePath: string) => Effect.Effect<void, EvalServiceValidationError>;
+    readonly inspect: (request: EvalComparisonRequest) => Effect.Effect<EvalSuiteInspection, EvalServiceValidationError>;
+    readonly estimate: (request: EvalComparisonRequest, mode: EvalComparisonMode) => Effect.Effect<EvalComparisonEstimate, EvalServiceEstimateError | EvalServiceValidationError>;
+    readonly runComparison: (request: EvalComparisonRequest, mode: EvalComparisonMode) => Effect.Effect<EvalComparisonResult, EvalServiceComparisonError | EvalServiceSpendLimitError | EvalServiceValidationError>;
+    readonly qualifyDimensionMatrix: (input: DimensionMatrixQualificationInput) => Effect.Effect<DimensionMatrixQualificationResult, EvalServiceError>;
+};
+declare const EvalService_base: Context.ServiceClass<EvalService, "@velum-labs/routekit-eval-service/EvalService", EvalServiceShape>;
+export declare class EvalService extends EvalService_base {
+}
+export declare const makeEvalService: (configuration?: EvalServiceConfiguration) => Effect.Effect<EvalService["Service"], never, EvalEngine | FileSystem.FileSystem | Path.Path>;
+export declare const makeEvalServiceLayer: (configuration?: EvalServiceConfiguration) => Layer.Layer<EvalService, never, EvalEngine | FileSystem.FileSystem | Path.Path>;
+export {};

package/dist/service.js

@@ -0,0 +1,345 @@
+import { assertExplicitEvalModel, assertRoutingBasis, EvalRunManifest } from "@velum-labs/routekit-eval-contracts";
+import { EvalEngine } from "@velum-labs/routekit-eval-engine";
+import { makeRoutingActivationStore } from "@velum-labs/routekit-eval-store";
+import { DEFAULT_MODEL_PRICING, PRICING_ALIASES } from "@velum-labs/routekit-registry";
+import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect";
+import { compileDimensionEvidenceMatrix } from "./dimension-evidence.js";
+import { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
+export class EvalService extends Context.Service()("@velum-labs/routekit-eval-service/EvalService") {
+}
+const detailOf = (cause) => cause instanceof Error ? cause.message : String(cause);
+const validatePositiveOption = (label, value, allowZero = false) => {
+    if (value === undefined)
+        return;
+    if (!Number.isFinite(value) || (allowZero ? value < 0 : value < 1)) {
+        throw new Error(`${label} must be ${allowZero ? "non-negative" : "at least 1"}`);
+    }
+};
+const validateGatewayUrl = (gatewayUrl) => {
+    const gateway = new URL(gatewayUrl);
+    if (gateway.protocol !== "http:" && gateway.protocol !== "https:") {
+        throw new Error("gatewayUrl must use http or https");
+    }
+    if (gateway.username.length > 0 || gateway.password.length > 0) {
+        throw new Error("gatewayUrl must not contain credentials");
+    }
+};
+const validateConfiguration = (configuration) => {
+    if (configuration.gatewayUrl === undefined) {
+        throw new Error("gatewayUrl is required for dimension matrix qualification");
+    }
+    validateGatewayUrl(configuration.gatewayUrl);
+    if (configuration.snapshotRoot === undefined || configuration.snapshotRoot.trim().length === 0) {
+        throw new Error("snapshotRoot is required for dimension matrix qualification");
+    }
+    validatePositiveOption("full.concurrency", configuration.full?.concurrency);
+    validatePositiveOption("full.timeoutMs", configuration.full?.timeoutMs);
+    return {
+        gatewayUrl: configuration.gatewayUrl,
+        snapshotRoot: configuration.snapshotRoot
+    };
+};
+const sameStrings = (left, right) => left.length === right.length && left.every((value, index) => value === right[index]);
+const validateDimensionMatrixInput = (configuration, input) => {
+    const validatedConfiguration = validateConfiguration(configuration);
+    assertRoutingBasis(input.basis);
+    if (input.candidateModels.length < 2) {
+        throw new Error("dimension matrix requires at least two candidate models");
+    }
+    if (new Set(input.candidateModels).size !== input.candidateModels.length) {
+        throw new Error("dimension matrix candidate models must be unique");
+    }
+    for (const model of input.candidateModels)
+        assertExplicitEvalModel(model, "candidate");
+    assertExplicitEvalModel(input.classifierModel, "classifier");
+    assertExplicitEvalModel(input.judgeModel, "judge");
+    const expectedDimensions = new Set(input.basis.dimensions.map((dimension) => dimension.id));
+    const suites = new Map();
+    for (const entry of input.suites) {
+        if (!expectedDimensions.has(entry.dimensionId)) {
+            throw new Error(`dimension matrix contains unknown dimension ${JSON.stringify(entry.dimensionId)}`);
+        }
+        if (suites.has(entry.dimensionId)) {
+            throw new Error(`dimension matrix contains duplicate dimension ${JSON.stringify(entry.dimensionId)}`);
+        }
+        if (entry.suitePath.trim().length === 0) {
+            throw new Error(`dimension matrix suite path is empty for ${JSON.stringify(entry.dimensionId)}`);
+        }
+        suites.set(entry.dimensionId, entry);
+    }
+    for (const dimensionId of expectedDimensions) {
+        if (!suites.has(dimensionId)) {
+            throw new Error(`dimension matrix is missing dimension ${JSON.stringify(dimensionId)}`);
+        }
+    }
+    return { ...validatedConfiguration, suites };
+};
+const comparisonRequest = (configuration, gatewayUrl, input, suite) => ({
+    version: 1,
+    profileId: suite.dimensionId,
+    suitePath: suite.suitePath,
+    candidateModels: [...input.candidateModels],
+    judgeModel: input.judgeModel,
+    gatewayUrl,
+    ...(configuration.full?.concurrency === undefined
+        ? {}
+        : { concurrency: configuration.full.concurrency }),
+    ...(configuration.full?.timeoutMs === undefined
+        ? {}
+        : { timeoutMs: configuration.full.timeoutMs })
+});
+const MAXIMUM_INPUT_TOKENS_PER_CALL = 256 * 1024;
+const modelPricing = (model) => {
+    const names = [model, model.includes("/") ? model.slice(model.indexOf("/") + 1) : model];
+    for (const name of names) {
+        const direct = Object.entries(DEFAULT_MODEL_PRICING).find(([candidate]) => candidate.toLowerCase() === name.toLowerCase())?.[1];
+        if (direct !== undefined)
+            return direct;
+        const alias = Object.entries(PRICING_ALIASES).find(([candidate]) => candidate.toLowerCase() === name.toLowerCase())?.[1];
+        if (alias === undefined)
+            continue;
+        const resolved = Object.entries(DEFAULT_MODEL_PRICING).find(([candidate]) => candidate.toLowerCase() === alias.toLowerCase())?.[1];
+        if (resolved !== undefined)
+            return resolved;
+    }
+    return undefined;
+};
+const maximumCallCost = (pricing, maximumOutputTokens) => (MAXIMUM_INPUT_TOKENS_PER_CALL * pricing.inputPer1mTokens +
+    maximumOutputTokens * pricing.outputPer1mTokens) /
+    1_000_000;
+const estimateManifest = (manifest) => {
+    const candidatePrices = manifest.candidateModels.map(modelPricing);
+    const judgePrice = modelPricing(manifest.judgeModel);
+    if (candidatePrices.some((pricing) => pricing === undefined) || judgePrice === undefined) {
+        return {
+            callCount: manifest.expectedCallCount,
+            pricingKnown: false
+        };
+    }
+    const candidateCost = candidatePrices.reduce((total, pricing) => total + manifest.caseCount * maximumCallCost(pricing, manifest.maxOutputTokens), 0);
+    const judgeCost = manifest.caseCount *
+        manifest.candidateModels.length *
+        maximumCallCost(judgePrice, manifest.maxOutputTokens);
+    return {
+        callCount: manifest.expectedCallCount,
+        maximumCostUsd: candidateCost + judgeCost,
+        pricingKnown: true
+    };
+};
+const enforceSpendLimit = (request, manifest) => {
+    if (request.spendLimitUsd === undefined)
+        return Effect.void;
+    const estimate = estimateManifest(manifest);
+    if (!estimate.pricingKnown || estimate.maximumCostUsd === undefined) {
+        return Effect.fail(new EvalServiceSpendLimitError({
+            operation: "enforce the comparison spend limit",
+            detail: "RouteKit Eval cannot enforce spendLimitUsd because pricing is unknown for one or more manifest models."
+        }));
+    }
+    if (estimate.maximumCostUsd > request.spendLimitUsd) {
+        return Effect.fail(new EvalServiceSpendLimitError({
+            operation: "enforce the comparison spend limit",
+            detail: `RouteKit Eval maximum estimated cost $${estimate.maximumCostUsd.toFixed(6)} ` +
+                `exceeds spendLimitUsd $${request.spendLimitUsd.toFixed(6)}.`
+        }));
+    }
+    return Effect.void;
+};
+const loadExecutionManifest = Effect.fn("EvalService.loadExecutionManifest")(function* (validation, request) {
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    const manifests = yield* fs
+        .glob("**/routekit.eval-manifest.json", {
+        root: validation.workingDirectory
+    })
+        .pipe(Effect.mapError((cause) => new EvalServiceValidationError({
+        operation: "discover the comparison manifest",
+        detail: detailOf(cause),
+        cause
+    })));
+    if (manifests.length !== 1 || manifests[0] === undefined) {
+        return yield* new EvalServiceValidationError({
+            operation: "discover the comparison manifest",
+            detail: "comparison requires exactly one routekit.eval-manifest.json"
+        });
+    }
+    const manifestPath = paths.isAbsolute(manifests[0])
+        ? manifests[0]
+        : paths.join(validation.workingDirectory, manifests[0]);
+    const raw = yield* fs.readFileString(manifestPath).pipe(Effect.mapError((cause) => new EvalServiceValidationError({
+        operation: "read the comparison manifest",
+        detail: detailOf(cause),
+        cause
+    })));
+    const json = yield* Effect.try({
+        try: () => JSON.parse(raw),
+        catch: (cause) => new EvalServiceValidationError({
+            operation: "parse the comparison manifest",
+            detail: "comparison manifest is not JSON",
+            cause
+        })
+    });
+    const manifest = yield* Schema.decodeUnknownEffect(EvalRunManifest)(json).pipe(Effect.mapError((cause) => new EvalServiceValidationError({
+        operation: "decode the comparison manifest",
+        detail: "comparison manifest is invalid",
+        cause
+    })));
+    if (manifest.profileId !== request.profileId ||
+        !sameStrings(manifest.candidateModels, request.candidateModels) ||
+        manifest.judgeModel !== request.judgeModel) {
+        return yield* new EvalServiceValidationError({
+            operation: "bind the comparison manifest",
+            detail: "comparison request profile or models do not match the authoritative manifest"
+        });
+    }
+    if (manifest.profileId.trim().length === 0 ||
+        manifest.caseCount < 1 ||
+        manifest.caseIds.length !== manifest.caseCount ||
+        new Set(manifest.caseIds).size !== manifest.caseIds.length ||
+        manifest.caseIds.some((caseId) => caseId.trim().length === 0)) {
+        return yield* new EvalServiceValidationError({
+            operation: "validate the comparison manifest cases",
+            detail: "comparison manifest case identities are incomplete or duplicated"
+        });
+    }
+    const expectedCallCount = manifest.caseCount * manifest.candidateModels.length * 2;
+    if (manifest.expectedCallCount !== expectedCallCount || manifest.maxOutputTokens < 1) {
+        return yield* new EvalServiceValidationError({
+            operation: "validate the comparison manifest limits",
+            detail: "comparison manifest call or output-token limits are inconsistent"
+        });
+    }
+    return {
+        suiteDigest: validation.suiteDigest,
+        manifest
+    };
+});
+export const makeEvalService = (configuration = {}) => Effect.gen(function* () {
+    const engine = yield* EvalEngine;
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    const inspectRaw = Effect.fn("EvalService.inspectRaw")(function* (request) {
+        const validation = yield* engine.validate(request.suitePath);
+        return yield* loadExecutionManifest(validation, request).pipe(Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
+    });
+    const validate = Effect.fn("EvalService.validate")(function* (suitePath) {
+        yield* engine.validate(suitePath).pipe(Effect.mapError((cause) => new EvalServiceValidationError({
+            operation: "validate the comparison suite",
+            detail: detailOf(cause),
+            cause
+        })));
+    });
+    const inspect = (request) => inspectRaw(request).pipe(Effect.mapError((cause) => cause instanceof EvalServiceValidationError
+        ? cause
+        : new EvalServiceValidationError({
+            operation: "inspect the comparison manifest",
+            detail: detailOf(cause),
+            cause
+        })));
+    const estimate = (request) => inspectRaw(request).pipe(Effect.map((inspection) => estimateManifest(inspection.manifest)), Effect.mapError((cause) => new EvalServiceEstimateError({
+        operation: "estimate the comparison",
+        detail: detailOf(cause),
+        cause
+    })));
+    const runInspectedComparison = (request, inspection) => enforceSpendLimit(request, inspection.manifest).pipe(Effect.flatMap(() => engine
+        .runComparison({
+        ...request,
+        expectedCaseIds: [...inspection.manifest.caseIds],
+        expectedCallCount: inspection.manifest.expectedCallCount,
+        maxOutputTokens: inspection.manifest.maxOutputTokens,
+        suiteDigest: inspection.suiteDigest
+    })
+        .pipe(Effect.mapError((cause) => new EvalServiceComparisonError({
+        operation: "run the comparison",
+        detail: detailOf(cause),
+        cause
+    })))));
+    const runComparison = (request) => inspect(request).pipe(Effect.flatMap((inspection) => runInspectedComparison(request, inspection)));
+    const qualifyDimensionMatrix = (input) => Effect.gen(function* () {
+        const validated = yield* Effect.try({
+            try: () => validateDimensionMatrixInput(configuration, input),
+            catch: (cause) => new EvalServiceConfigurationError({
+                operation: "validate the dimension matrix qualification",
+                detail: detailOf(cause),
+                cause
+            })
+        });
+        const inspections = new Map();
+        for (const dimension of input.basis.dimensions) {
+            const suite = validated.suites.get(dimension.id);
+            const request = comparisonRequest(configuration, validated.gatewayUrl, input, suite);
+            const inspection = yield* inspect(request);
+            if (inspection.manifest.profileId !== dimension.id ||
+                inspection.manifest.judgeModel !== input.judgeModel ||
+                !sameStrings(inspection.manifest.candidateModels, input.candidateModels)) {
+                return yield* new EvalServiceValidationError({
+                    operation: "inspect the full comparison manifest",
+                    detail: `authoritative manifest does not match dimension ${JSON.stringify(dimension.id)}`
+                });
+            }
+            inspections.set(dimension.id, inspection);
+        }
+        const comparisons = [];
+        const evidenceInputs = [];
+        for (const dimension of input.basis.dimensions) {
+            const suite = validated.suites.get(dimension.id);
+            const inspection = inspections.get(dimension.id);
+            const request = comparisonRequest(configuration, validated.gatewayUrl, input, suite);
+            const comparison = yield* runInspectedComparison(request, inspection);
+            if (comparison.profileId !== dimension.id ||
+                comparison.judgeModel !== input.judgeModel ||
+                comparison.suiteDigest !== inspection.suiteDigest) {
+                return yield* new EvalServiceComparisonError({
+                    operation: "validate the full comparison",
+                    detail: `comparison identity does not match dimension ${JSON.stringify(dimension.id)}`
+                });
+            }
+            comparisons.push(comparison);
+            evidenceInputs.push({
+                dimensionId: dimension.id,
+                suiteDigest: inspection.suiteDigest,
+                judgeModel: inspection.manifest.judgeModel,
+                expectedCaseIds: inspection.manifest.caseIds,
+                comparison
+            });
+        }
+        const compiled = yield* Effect.try({
+            try: () => compileDimensionEvidenceMatrix({
+                basis: input.basis,
+                candidateModels: input.candidateModels,
+                comparisons: evidenceInputs
+            }),
+            catch: (cause) => new EvalServicePolicyError({
+                operation: "compile the dimension evidence matrix",
+                detail: detailOf(cause),
+                cause
+            })
+        });
+        const snapshot = yield* makeRoutingActivationStore(validated.snapshotRoot)
+            .publish({
+            basisDigest: input.basis.basisDigest,
+            evidenceDigest: compiled.evidenceDigest,
+            classifierModel: input.classifierModel,
+            objective: input.objective,
+            maximumUnknownWeight: input.maximumUnknownWeight,
+            ...(input.constraints === undefined ? {} : { constraints: input.constraints }),
+            dimensions: [...input.basis.dimensions],
+            candidateModels: [...input.candidateModels],
+            evidence: [...compiled.evidence]
+        })
+            .pipe(Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths), Effect.mapError((cause) => new EvalServicePublicationError({
+            operation: "publish the dimension evidence snapshot",
+            detail: detailOf(cause),
+            cause
+        })));
+        return { comparisons, snapshot };
+    });
+    return EvalService.of({
+        validate,
+        inspect,
+        estimate,
+        runComparison,
+        qualifyDimensionMatrix
+    });
+});
+export const makeEvalServiceLayer = (configuration = {}) => Layer.effect(EvalService, makeEvalService(configuration));

package/dist/test/dimension-evidence.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/dimension-evidence.test.js

@@ -0,0 +1,155 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { compileDimensionEvidenceMatrix, wilsonLowerBound95 } from "../dimension-evidence.js";
+const dimensionIds = [
+    "gateway-protocol",
+    "eval-routing",
+    "account-pooling",
+    "typescript-maintenance",
+    "release-operations"
+];
+const models = ["openai/model-a", "openai/model-b"];
+const caseIds = ["case-1", "case-2", "case-3", "case-4", "case-5"];
+const judgeModel = "openai/judge";
+function mutable(value) {
+    return structuredClone(value);
+}
+const basis = {
+    version: 2,
+    basisDigest: "definition-set-v2",
+    dimensions: dimensionIds.map((id) => ({
+        id,
+        description: `Requests about ${id}`,
+        includes: [`Tasks specifically involving ${id}`],
+        excludes: [`Tasks unrelated to ${id}`]
+    }))
+};
+function cases(options = {}) {
+    return caseIds.map((caseId, index) => {
+        const passed = !options.failed?.has(caseId);
+        return {
+            caseId,
+            outcome: passed ? "passed" : "failed",
+            measurement: {
+                judgeScore: passed ? 0.9 : 0.2,
+                ...(!options.unpriced?.has(caseId) ? { costUsd: 0.01 + index / 1_000 } : {}),
+                ...(!options.missingDuration?.has(caseId) ? { durationMs: 100 + index * 10 } : {}),
+                inputTokens: 100 + index,
+                outputTokens: 20 + index
+            }
+        };
+    });
+}
+function comparison(dimensionId) {
+    return {
+        version: 1,
+        comparisonId: `comparison-${dimensionId}`,
+        profileId: dimensionId,
+        suiteDigest: `suite-${dimensionId}`,
+        judgeModel,
+        startedAt: "2026-08-17T00:00:00.000Z",
+        finishedAt: "2026-08-17T00:01:00.000Z",
+        models: models.map((model) => ({ model, cases: cases() }))
+    };
+}
+function input() {
+    return mutable({
+        basis,
+        candidateModels: models,
+        comparisons: dimensionIds.map((dimensionId) => ({
+            dimensionId,
+            suiteDigest: `suite-${dimensionId}`,
+            judgeModel,
+            expectedCaseIds: caseIds,
+            comparison: comparison(dimensionId)
+        }))
+    });
+}
+test("complete comparison results compile into a deterministic model-dimension matrix", () => {
+    const first = compileDimensionEvidenceMatrix(input());
+    const secondInput = input();
+    secondInput.comparisons.reverse();
+    secondInput.comparisons.forEach((entry) => {
+        entry.comparison.models.reverse();
+        entry.comparison.models.forEach((model) => model.cases.reverse());
+    });
+    const second = compileDimensionEvidenceMatrix(secondInput);
+    assert.equal(first.evidence.length, dimensionIds.length * models.length);
+    assert.match(first.evidenceDigest, /^[a-f0-9]{64}$/u);
+    assert.equal(first.evidenceDigest, second.evidenceDigest);
+    assert.deepEqual(first.evidence, second.evidence);
+    const cell = first.evidence.find((entry) => entry.dimensionId === "gateway-protocol" && entry.model === "openai/model-a");
+    assert.equal(cell?.quality.sampleCount, 5);
+    assert.equal(cell?.quality.passRate, 1);
+    assert.ok((cell?.quality.lowerConfidenceBound ?? 0) > 0.56);
+    assert.ok((cell?.quality.lowerConfidenceBound ?? 1) < 0.57);
+    assert.equal(cell?.failureRate, 0);
+    assert.equal(cell?.averageJudgeScore, 0.9);
+    assert.equal(cell?.p95DurationMs, 140);
+    assert.equal(cell?.unpricedCalls, 0);
+    assert.equal(cell?.averageCostUsd, 0.012);
+});
+test("partial pricing is explicit and never converted into a known average", () => {
+    const value = input();
+    value.comparisons[0].comparison.models[0].cases = cases({
+        unpriced: new Set(["case-4", "case-5"])
+    });
+    const cell = compileDimensionEvidenceMatrix(value).evidence.find((entry) => entry.dimensionId === dimensionIds[0] && entry.model === models[0]);
+    assert.equal(cell?.unpricedCalls, 2);
+    assert.equal(cell?.averageCostUsd, undefined);
+});
+test("partial duration measurements do not produce a misleading percentile", () => {
+    const value = input();
+    value.comparisons[0].comparison.models[0].cases = cases({
+        missingDuration: new Set(["case-5"])
+    });
+    const cell = compileDimensionEvidenceMatrix(value).evidence.find((entry) => entry.dimensionId === dimensionIds[0] && entry.model === models[0]);
+    assert.equal(cell?.p95DurationMs, undefined);
+});
+test("matrix compilation rejects missing and duplicate dimensions or candidates", () => {
+    const missingDimension = input();
+    missingDimension.comparisons.pop();
+    assert.throws(() => compileDimensionEvidenceMatrix(missingDimension), /missing dimension/);
+    const duplicateDimension = input();
+    duplicateDimension.comparisons[1] = duplicateDimension.comparisons[0];
+    assert.throws(() => compileDimensionEvidenceMatrix(duplicateDimension), /duplicate dimension/);
+    const missingCandidate = input();
+    missingCandidate.comparisons[0].comparison.models.pop();
+    assert.throws(() => compileDimensionEvidenceMatrix(missingCandidate), /missing candidate/);
+    const unexpectedCandidate = input();
+    unexpectedCandidate.comparisons[0].comparison.models[0].model = "openai/unexpected";
+    assert.throws(() => compileDimensionEvidenceMatrix(unexpectedCandidate), /unexpected candidate/);
+});
+test("matrix compilation rejects incomplete or ambiguous case evidence", () => {
+    const missingCase = input();
+    missingCase.comparisons[0].comparison.models[0].cases.pop();
+    assert.throws(() => compileDimensionEvidenceMatrix(missingCase), /has 4 cases; expected 5/);
+    const duplicateCase = input();
+    duplicateCase.comparisons[0].comparison.models[0].cases[4] =
+        duplicateCase.comparisons[0].comparison.models[0].cases[0];
+    assert.throws(() => compileDimensionEvidenceMatrix(duplicateCase), /duplicate case/);
+    const missingJudgeScore = input();
+    delete missingJudgeScore.comparisons[0].comparison.models[0].cases[0].measurement.judgeScore;
+    assert.throws(() => compileDimensionEvidenceMatrix(missingJudgeScore), /missing a judge score/);
+    const cutoff = input();
+    cutoff.comparisons[0].comparison.models[0].cases[0].outcome = "cutoff";
+    assert.throws(() => compileDimensionEvidenceMatrix(cutoff), /non-terminal case/);
+});
+test("matrix compilation binds profile, suite digest, and judge", () => {
+    const wrongProfile = input();
+    wrongProfile.comparisons[0].comparison.profileId = "wrong";
+    assert.throws(() => compileDimensionEvidenceMatrix(wrongProfile), /does not match dimension/);
+    const wrongDigest = input();
+    wrongDigest.comparisons[0].comparison.suiteDigest = "wrong";
+    assert.throws(() => compileDimensionEvidenceMatrix(wrongDigest), /suite digest does not match/);
+    const wrongJudge = input();
+    wrongJudge.comparisons[0].comparison.judgeModel = "openai/wrong";
+    assert.throws(() => compileDimensionEvidenceMatrix(wrongJudge), /comparison judge does not match/);
+});
+test("Wilson lower confidence bounds are conservative and validate counts", () => {
+    assert.equal(wilsonLowerBound95(0, 5), 0);
+    assert.ok(wilsonLowerBound95(5, 5) < 1);
+    assert.ok(wilsonLowerBound95(19, 20) < 0.95);
+    assert.throws(() => wilsonLowerBound95(6, 5), /within the sample/);
+    assert.throws(() => wilsonLowerBound95(0, 0), /within the sample/);
+});

package/dist/test/production-runner.test.d.ts

@@ -0,0 +1 @@
+export {};