@velum-labs/routekit-eval-service 1.3.2 → 1.4.0

package/dist/layer-options.d.ts

@@ -4,11 +4,13 @@ export type RouteKitEvalGatewayCallEvent = {
     readonly observationId: string;
     readonly phase: "issued";
     readonly role: "candidate" | "judge";
+    readonly model: string;
 } | {
     readonly callId?: string;
     readonly observationId: string;
     readonly phase: "completed";
     readonly role: "candidate" | "judge";
+    readonly model: string;
 };
 export type RouteKitEvalServiceOptions = Omit<NodeTestExecutionOptions, "bridgeOrigin"> & {
     /**

package/dist/production-runner.js

@@ -1,6 +1,7 @@
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { EVAL_ATTRIBUTION_HEADER } from "@velum-labs/routekit-eval-contracts";
 import { EvalEngineExecutionError, EvalExecutionPort, makeEvalEngineLayer, makeRouteKitEvalExecutionPortService } from "@velum-labs/routekit-eval-engine";
+import { CostServiceLive } from "@velum-labs/routekit-registry";
 import { Data, Effect, FileSystem, Layer, Path, Stream } from "effect";
 import { HttpClient } from "effect/unstable/http";
 import { makeEvalServiceLayer } from "./service.js";
@@ -21,22 +22,42 @@ const gatewayCallRole = (raw) => {
         return undefined;
     }
 };
+const gatewayCallModel = (body) => {
+    const raw = body._tag === "Uint8Array"
+        ? new TextDecoder().decode(body.body)
+        : body._tag === "Raw" && typeof body.body === "string"
+            ? body.body
+            : undefined;
+    if (raw === undefined)
+        return undefined;
+    try {
+        const decoded = JSON.parse(raw);
+        return typeof decoded.model === "string" && decoded.model.length > 0
+            ? decoded.model
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
 const observeGatewayCalls = (client, observe) => {
     let nextObservationId = 0;
     return HttpClient.transform(client, (responseEffect, request) => {
         const role = gatewayCallRole(request.headers[EVAL_ATTRIBUTION_HEADER]);
-        if (role === undefined)
+        const model = gatewayCallModel(request.body);
+        if (role === undefined || model === undefined)
             return responseEffect;
         return Effect.gen(function* () {
             nextObservationId += 1;
             const observationId = `gateway-call-${String(nextObservationId)}`;
-            yield* observe({ observationId, phase: "issued", role });
+            yield* observe({ observationId, phase: "issued", role, model });
             const response = yield* responseEffect;
             const rawCallId = response.headers[MODEL_CALL_ID_HEADER]?.trim();
             yield* observe({
                 observationId,
                 phase: "completed",
                 role,
+                model,
                 ...(rawCallId === undefined || rawCallId.length === 0 ? {} : { callId: rawCallId })
             });
             return response;
@@ -112,4 +133,4 @@ const makeProductionEvalEngineLayer = (options) => makeEvalEngineLayer().pipe(La
  * execution port is the only adapter; no parallel comparison-runner service
  * mirrors or wraps the engine API.
  */
-export const makeRouteKitEvalServiceLayer = (configuration, options) => makeEvalServiceLayer(configuration).pipe(Layer.provide(makeProductionEvalEngineLayer(options)), Layer.provide(NodeServicesLayer));
+export const makeRouteKitEvalServiceLayer = (configuration, options) => makeEvalServiceLayer(configuration).pipe(Layer.provide(makeProductionEvalEngineLayer(options)), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));

package/dist/service.d.ts

@@ -1,6 +1,7 @@
 import type { EvalComparisonRequest, EvalComparisonResult, EvalReasoningEffort, PublishedRoutingActivation, RoutingActivationConstraints, RoutingBasis, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
 import { EvalRunManifest } from "@velum-labs/routekit-eval-contracts";
 import { EvalEngine } from "@velum-labs/routekit-eval-engine";
+import { CostService } from "@velum-labs/routekit-registry";
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
 export type EvalComparisonMode = "pilot" | "full";
@@ -10,8 +11,13 @@ export type EvalSuiteInspection = {
 };
 export type EvalComparisonEstimate = {
     readonly callCount: number;
+    readonly maximumInputTokens: number;
+    readonly maximumOutputTokens: number;
     readonly maximumCostUsd?: number;
-    readonly pricingKnown: boolean;
+    readonly knownPricedMaximumUsd: number;
+    readonly unpricedCalls: number;
+    readonly subscriptionUnitCalls: number;
+    readonly unknowns: readonly string[];
 };
 export type EvalRunConfiguration = {
     readonly concurrency?: number;
@@ -52,6 +58,6 @@ export type EvalServiceShape = {
 declare const EvalService_base: Context.ServiceClass<EvalService, "@velum-labs/routekit-eval-service/EvalService", EvalServiceShape>;
 export declare class EvalService extends EvalService_base {
 }
-export declare const makeEvalService: (configuration?: EvalServiceConfiguration) => Effect.Effect<EvalService["Service"], never, EvalEngine | FileSystem.FileSystem | Path.Path>;
-export declare const makeEvalServiceLayer: (configuration?: EvalServiceConfiguration) => Layer.Layer<EvalService, never, EvalEngine | FileSystem.FileSystem | Path.Path>;
+export declare const makeEvalService: (configuration?: EvalServiceConfiguration) => Effect.Effect<EvalService["Service"], never, CostService | EvalEngine | FileSystem.FileSystem | Path.Path>;
+export declare const makeEvalServiceLayer: (configuration?: EvalServiceConfiguration) => Layer.Layer<EvalService, never, CostService | EvalEngine | FileSystem.FileSystem | Path.Path>;
 export {};

package/dist/service.js

@@ -1,7 +1,7 @@
 import { assertExplicitEvalModel, assertRoutingBasis, EvalRunManifest } from "@velum-labs/routekit-eval-contracts";
 import { EvalEngine } from "@velum-labs/routekit-eval-engine";
 import { makeRoutingActivationStore } from "@velum-labs/routekit-eval-store";
-import { DEFAULT_MODEL_PRICING, PRICING_ALIASES } from "@velum-labs/routekit-registry";
+import { CostService } from "@velum-labs/routekit-registry";
 import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect";
 import { compileDimensionEvidenceMatrix } from "./dimension-evidence.js";
 import { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
@@ -101,61 +101,57 @@ const comparisonRequest = (configuration, gatewayUrl, input, suite) => ({
         : { timeoutMs: configuration.full.timeoutMs })
 });
 const MAXIMUM_INPUT_TOKENS_PER_CALL = 256 * 1024;
-const modelPricing = (model) => {
-    const names = [model, model.includes("/") ? model.slice(model.indexOf("/") + 1) : model];
-    for (const name of names) {
-        const direct = Object.entries(DEFAULT_MODEL_PRICING).find(([candidate]) => candidate.toLowerCase() === name.toLowerCase())?.[1];
-        if (direct !== undefined)
-            return direct;
-        const alias = Object.entries(PRICING_ALIASES).find(([candidate]) => candidate.toLowerCase() === name.toLowerCase())?.[1];
-        if (alias === undefined)
-            continue;
-        const resolved = Object.entries(DEFAULT_MODEL_PRICING).find(([candidate]) => candidate.toLowerCase() === alias.toLowerCase())?.[1];
-        if (resolved !== undefined)
-            return resolved;
-    }
-    return undefined;
-};
-const maximumCallCost = (pricing, maximumOutputTokens) => (MAXIMUM_INPUT_TOKENS_PER_CALL * pricing.inputPer1mTokens +
-    maximumOutputTokens * pricing.outputPer1mTokens) /
-    1_000_000;
-const estimateManifest = (manifest) => {
-    const candidatePrices = manifest.candidateModels.map(modelPricing);
-    const judgePrice = modelPricing(manifest.judgeModel);
-    if (candidatePrices.some((pricing) => pricing === undefined) || judgePrice === undefined) {
-        return {
-            callCount: manifest.expectedCallCount,
-            pricingKnown: false
-        };
-    }
-    const candidateCost = candidatePrices.reduce((total, pricing) => total + manifest.caseCount * maximumCallCost(pricing, manifest.maxOutputTokens), 0);
-    const judgeCost = manifest.caseCount *
-        manifest.candidateModels.length *
-        maximumCallCost(judgePrice, manifest.maxOutputTokens);
-    return {
+const estimateManifest = (manifest, costService) => {
+    const calls = [
+        ...manifest.candidateModels.map((model) => ({
+            model,
+            callCount: manifest.caseCount,
+            maximumInputTokens: MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: manifest.maxOutputTokens,
+            ...(manifest.candidateReasoningEfforts?.[model] === undefined
+                ? {}
+                : { reasoningEffort: manifest.candidateReasoningEfforts[model] })
+        })),
+        {
+            model: manifest.judgeModel,
+            callCount: manifest.caseCount * manifest.candidateModels.length,
+            maximumInputTokens: MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: manifest.maxOutputTokens
+        }
+    ];
+    return costService.estimate(calls).pipe(Effect.map((estimate) => ({
         callCount: manifest.expectedCallCount,
-        maximumCostUsd: candidateCost + judgeCost,
-        pricingKnown: true
-    };
+        maximumInputTokens: estimate.knownInputTokens,
+        maximumOutputTokens: estimate.knownOutputTokens,
+        ...(estimate.maximumCostUsd === undefined ? {} : { maximumCostUsd: estimate.maximumCostUsd }),
+        knownPricedMaximumUsd: estimate.knownPricedMaximumUsd,
+        unpricedCalls: estimate.unpricedCalls + estimate.upstreamManagedCalls + estimate.unknownBillingCalls,
+        subscriptionUnitCalls: estimate.subscriptionUnitCalls,
+        unknowns: estimate.unknowns
+    })));
 };
-const enforceSpendLimit = (request, manifest) => {
-    if (request.spendLimitUsd === undefined)
+const enforceSpendLimit = (request, manifest, costService) => {
+    const spendLimitUsd = request.spendLimitUsd;
+    if (spendLimitUsd === undefined)
         return Effect.void;
-    const estimate = estimateManifest(manifest);
-    if (!estimate.pricingKnown || estimate.maximumCostUsd === undefined) {
-        return Effect.fail(new EvalServiceSpendLimitError({
-            operation: "enforce the comparison spend limit",
-            detail: "RouteKit Eval cannot enforce spendLimitUsd because pricing is unknown for one or more manifest models."
-        }));
-    }
-    if (estimate.maximumCostUsd > request.spendLimitUsd) {
-        return Effect.fail(new EvalServiceSpendLimitError({
-            operation: "enforce the comparison spend limit",
-            detail: `RouteKit Eval maximum estimated cost $${estimate.maximumCostUsd.toFixed(6)} ` +
-                `exceeds spendLimitUsd $${request.spendLimitUsd.toFixed(6)}.`
-        }));
-    }
-    return Effect.void;
+    return estimateManifest(manifest, costService).pipe(Effect.flatMap((estimate) => {
+        if (estimate.unpricedCalls > 0 ||
+            estimate.maximumCostUsd === undefined ||
+            estimate.unknowns.length > 0) {
+            return Effect.fail(new EvalServiceSpendLimitError({
+                operation: "enforce the comparison spend limit",
+                detail: "RouteKit Eval cannot enforce spendLimitUsd because pricing is unknown for one or more manifest models."
+            }));
+        }
+        if (estimate.maximumCostUsd > spendLimitUsd) {
+            return Effect.fail(new EvalServiceSpendLimitError({
+                operation: "enforce the comparison spend limit",
+                detail: `RouteKit Eval maximum estimated cost $${estimate.maximumCostUsd.toFixed(6)} ` +
+                    `exceeds spendLimitUsd $${spendLimitUsd.toFixed(6)}.`
+            }));
+        }
+        return Effect.void;
+    }));
 };
 const EXECUTION_MANIFEST_NAME = "routekit.eval-manifest.json";
 const discoverExecutionManifests = Effect.fnUntraced(function* (root) {
@@ -253,6 +249,7 @@ const loadExecutionManifest = Effect.fn("EvalService.loadExecutionManifest")(fun
 });
 export const makeEvalService = (configuration = {}) => Effect.gen(function* () {
     const engine = yield* EvalEngine;
+    const costService = yield* CostService;
     const fs = yield* FileSystem.FileSystem;
     const paths = yield* Path.Path;
     const inspectRaw = Effect.fn("EvalService.inspectRaw")(function* (request) {
@@ -273,12 +270,12 @@ export const makeEvalService = (configuration = {}) => Effect.gen(function* () {
             detail: detailOf(cause),
             cause
         })));
-    const estimate = (request) => inspectRaw(request).pipe(Effect.map((inspection) => estimateManifest(inspection.manifest)), Effect.mapError((cause) => new EvalServiceEstimateError({
+    const estimate = (request) => inspectRaw(request).pipe(Effect.flatMap((inspection) => estimateManifest(inspection.manifest, costService)), Effect.mapError((cause) => new EvalServiceEstimateError({
         operation: "estimate the comparison",
         detail: detailOf(cause),
         cause
     })));
-    const runInspectedComparison = (request, inspection) => enforceSpendLimit(request, inspection.manifest).pipe(Effect.flatMap(() => engine
+    const runInspectedComparison = (request, inspection) => enforceSpendLimit(request, inspection.manifest, costService).pipe(Effect.flatMap(() => engine
         .runComparison({
         ...request,
         expectedCaseIds: [...inspection.manifest.caseIds],

package/dist/test/production-runner.test.js

@@ -99,7 +99,10 @@ test("production runner estimates the exact imported nested-loop testdrive suite
         }, "pilot");
     })).pipe(Effect.provide(NodeHttpClient.layerUndici)));
     assert.equal(result.callCount, 30);
-    assert.equal(result.pricingKnown, true);
+    assert.equal(result.unpricedCalls, 0);
+    assert.equal(result.subscriptionUnitCalls, 0);
+    assert.equal(result.maximumInputTokens, 30 * 256 * 1024);
+    assert.equal(result.maximumOutputTokens, 30 * 1_024);
     assert.equal((result.maximumCostUsd ?? 0) > 0, true);
 });
 test("production runner rejects a manifest for a different profile", async () => {
@@ -234,6 +237,7 @@ test("production runner executes candidate and judge traffic through the live en
                 observed.push({
                     phase: event.phase,
                     role: event.role,
+                    model: event.model,
                     ...(event.phase === "completed" && event.callId !== undefined
                         ? { callId: event.callId }
                         : {})
@@ -251,17 +255,19 @@ test("production runner executes candidate and judge traffic through the live en
             { model: "openai/judge", reasoningEffort: undefined }
         ]);
         assert.deepEqual(observed, [
-            { phase: "issued", role: "candidate" },
+            { phase: "issued", role: "candidate", model: "openai/cheap" },
             {
                 callId: "model-call-1",
                 phase: "completed",
-                role: "candidate"
+                role: "candidate",
+                model: "openai/cheap"
             },
-            { phase: "issued", role: "judge" },
+            { phase: "issued", role: "judge", model: "openai/judge" },
             {
                 callId: "model-call-2",
                 phase: "completed",
-                role: "judge"
+                role: "judge",
+                model: "openai/judge"
             }
         ]);
         assert.deepEqual(result.models.map(({ model }) => model), ["openai/cheap"]);

package/dist/test/service.test.js

@@ -5,6 +5,7 @@ import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { EvalEngine } from "@velum-labs/routekit-eval-engine";
+import { CostServiceLive } from "@velum-labs/routekit-registry";
 import { Effect, Fiber, FileSystem, Layer, Path } from "effect";
 import { EvalService, makeEvalService, makeEvalServiceLayer } from "../service.js";
 const roots = [];
@@ -135,7 +136,7 @@ test("manifest discovery does not call FileSystem.glob when Node glob is unavail
                 suiteDigest: "suite-nested"
             }),
             runComparison: () => Effect.die("unexpected comparison")
-        })), Effect.provideService(FileSystem.FileSystem, fileSystemWithoutNodeGlob), Effect.provideService(Path.Path, paths));
+        })), Effect.provideService(FileSystem.FileSystem, fileSystemWithoutNodeGlob), Effect.provideService(Path.Path, paths), Effect.provide(CostServiceLive));
         return yield* service.inspect({
             version: 1,
             profileId: "nested",
@@ -162,7 +163,7 @@ test("dimension matrix qualification inspects every manifest before publishing o
     const layer = makeEvalServiceLayer({
         gatewayUrl: "http://127.0.0.1:8080/v1",
         snapshotRoot
-    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    }).pipe(Layer.provide(engine), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
     const result = await Effect.runPromise(Effect.gen(function* () {
         return yield* (yield* EvalService).qualifyDimensionMatrix(qualification(root));
     }).pipe(Effect.provide(layer)));
@@ -184,7 +185,7 @@ test("dimension matrix qualification never publishes incomplete comparison evide
     const layer = makeEvalServiceLayer({
         gatewayUrl: "http://127.0.0.1:8080/v1",
         snapshotRoot
-    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    }).pipe(Layer.provide(engine), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
     const exit = await Effect.runPromiseExit(Effect.gen(function* () {
         return yield* (yield* EvalService).qualifyDimensionMatrix(qualification(root));
     }).pipe(Effect.provide(layer)));
@@ -204,7 +205,7 @@ test("interrupted qualification leaves no half-published activation for a daemon
     const layer = makeEvalServiceLayer({
         gatewayUrl: "http://127.0.0.1:8080/v1",
         snapshotRoot
-    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    }).pipe(Layer.provide(engine), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
     await Effect.runPromise(Effect.gen(function* () {
         const fiber = yield* Effect.forkChild((yield* EvalService).qualifyDimensionMatrix(qualification(root)));
         yield* Effect.yieldNow;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.3.2",
+  "version": "1.4.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.3.2",
-    "@velum-labs/routekit-eval-engine": "1.3.2",
-    "@velum-labs/routekit-eval-store": "1.3.2",
-    "@velum-labs/routekit-registry": "1.3.2"
+    "@velum-labs/routekit-eval-contracts": "1.4.0",
+    "@velum-labs/routekit-eval-engine": "1.4.0",
+    "@velum-labs/routekit-eval-store": "1.4.0",
+    "@velum-labs/routekit-registry": "1.4.0"
   },
   "devDependencies": {
     "typescript": "7.0.2"