@velum-labs/routekit-eval-service 1.1.1 → 1.3.0

package/dist/service.d.ts

@@ -1,4 +1,4 @@
-import type { EvalComparisonRequest, EvalComparisonResult, PublishedRoutingActivation, RoutingActivationConstraints, RoutingBasis, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
+import type { EvalComparisonRequest, EvalComparisonResult, EvalReasoningEffort, PublishedRoutingActivation, RoutingActivationConstraints, RoutingBasis, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
 import { EvalRunManifest } from "@velum-labs/routekit-eval-contracts";
 import { EvalEngine } from "@velum-labs/routekit-eval-engine";
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
@@ -29,6 +29,7 @@ export type DimensionMatrixSuite = {
 export type DimensionMatrixQualificationInput = {
     readonly basis: RoutingBasis;
     readonly candidateModels: ReadonlyArray<string>;
+    readonly candidateReasoningEfforts?: Readonly<Record<string, EvalReasoningEffort>>;
     readonly classifierModel: string;
     readonly judgeModel: string;
     readonly objective: RoutingObjectivePolicy;

package/dist/service.js

@@ -40,6 +40,12 @@ const validateConfiguration = (configuration) => {
     };
 };
 const sameStrings = (left, right) => left.length === right.length && left.every((value, index) => value === right[index]);
+const sameReasoningEfforts = (left, right) => {
+    const leftEntries = Object.entries(left ?? {});
+    const rightEntries = Object.entries(right ?? {});
+    return (leftEntries.length === rightEntries.length &&
+        leftEntries.every(([model, effort]) => right?.[model] === effort));
+};
 const validateDimensionMatrixInput = (configuration, input) => {
     const validatedConfiguration = validateConfiguration(configuration);
     assertRoutingBasis(input.basis);
@@ -51,6 +57,9 @@ const validateDimensionMatrixInput = (configuration, input) => {
     }
     for (const model of input.candidateModels)
         assertExplicitEvalModel(model, "candidate");
+    if (Object.keys(input.candidateReasoningEfforts ?? {}).some((model) => !input.candidateModels.includes(model))) {
+        throw new Error("dimension matrix reasoning efforts must belong to candidate models");
+    }
     assertExplicitEvalModel(input.classifierModel, "classifier");
     assertExplicitEvalModel(input.judgeModel, "judge");
     const expectedDimensions = new Set(input.basis.dimensions.map((dimension) => dimension.id));
@@ -79,6 +88,9 @@ const comparisonRequest = (configuration, gatewayUrl, input, suite) => ({
     profileId: suite.dimensionId,
     suitePath: suite.suitePath,
     candidateModels: [...input.candidateModels],
+    ...(input.candidateReasoningEfforts === undefined
+        ? {}
+        : { candidateReasoningEfforts: input.candidateReasoningEfforts }),
     judgeModel: input.judgeModel,
     gatewayUrl,
     ...(configuration.full?.concurrency === undefined
@@ -210,6 +222,7 @@ const loadExecutionManifest = Effect.fn("EvalService.loadExecutionManifest")(fun
     })));
     if (manifest.profileId !== request.profileId ||
         !sameStrings(manifest.candidateModels, request.candidateModels) ||
+        !sameReasoningEfforts(manifest.candidateReasoningEfforts, request.candidateReasoningEfforts) ||
         manifest.judgeModel !== request.judgeModel) {
         return yield* new EvalServiceValidationError({
             operation: "bind the comparison manifest",

package/dist/test/production-runner.test.js

@@ -194,7 +194,7 @@ test("production runner executes candidate and judge traffic through the live en
         'import { setupAgent, setupJudge } from "routekit/eval";',
         'const judge = setupJudge({ agent: setupAgent({ model: "openai/judge" }), minScore: 0.8 });',
         'test("support case", async () => {',
-        '  const run = await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        '  const run = await setupAgent({ model: "openai/cheap", parameters: { reasoning: { effort: "low" } } }).run({ prompt: "Help", caseId: "support-case" });',
         "  run.toComplete();",
         '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
         "});"
@@ -208,7 +208,8 @@ test("production runner executes candidate and judge traffic through the live en
             calls.push({
                 authorization: incoming.headers.authorization,
                 maxOutputTokens: body.max_completion_tokens,
-                model: body.model
+                model: body.model,
+                reasoningEffort: body.reasoning_effort
             });
             const content = body.model === "openai/judge"
                 ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
@@ -245,6 +246,10 @@ test("production runner executes candidate and judge traffic through the live en
         assert.deepEqual(calls.map(({ model }) => model), ["openai/cheap", "openai/judge"]);
         assert.equal(calls.every(({ authorization }) => authorization === "Bearer parent-only-token"), true);
         assert.equal(calls.every(({ maxOutputTokens }) => maxOutputTokens === 1_024), true);
+        assert.deepEqual(calls.map(({ model, reasoningEffort }) => ({ model, reasoningEffort })), [
+            { model: "openai/cheap", reasoningEffort: "low" },
+            { model: "openai/judge", reasoningEffort: undefined }
+        ]);
         assert.deepEqual(observed, [
             { phase: "issued", role: "candidate" },
             {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.1.1",
+  "version": "1.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.1.1",
-    "@velum-labs/routekit-eval-engine": "1.1.1",
-    "@velum-labs/routekit-eval-store": "1.1.1",
-    "@velum-labs/routekit-registry": "1.1.1"
+    "@velum-labs/routekit-eval-contracts": "1.3.0",
+    "@velum-labs/routekit-eval-engine": "1.3.0",
+    "@velum-labs/routekit-eval-store": "1.3.0",
+    "@velum-labs/routekit-registry": "1.3.0"
   },
   "devDependencies": {
     "typescript": "7.0.2"