@velum-labs/routekit-eval-service 1.0.11 → 1.0.12

package/dist/layer-options.d.ts

@@ -7,4 +7,12 @@ export type RouteKitEvalServiceOptions = Omit<NodeTestExecutionOptions, "bridgeO
      * execution fails before starting a child when it is absent.
      */
     readonly bearerCredential?: string;
+    /**
+     * Host-owned per-test deadline.
+     *
+     * When set, the production execution adapter applies this value at the
+     * node:test boundary even if an intermediate comparison request carries a
+     * stale or missing timeout.
+     */
+    readonly timeoutMs?: number;
 };

package/dist/production-runner.js

@@ -20,13 +20,24 @@ const makeProductionExecutionPort = (options) => Effect.map(HttpClient.HttpClien
             }))
         };
     }
-    return makeRouteKitEvalExecutionPortService({
+    const execution = makeRouteKitEvalExecutionPortService({
         bearerCredential,
         ...(options.childEnvironment === undefined
             ? {}
             : { childEnvironment: options.childEnvironment }),
         ...(options.execPath === undefined ? {} : { execPath: options.execPath })
     }, httpClient);
+    if (options.timeoutMs === undefined)
+        return execution;
+    return {
+        execute: (input) => execution.execute({
+            ...input,
+            request: {
+                ...input.request,
+                timeoutMs: options.timeoutMs
+            }
+        })
+    };
 });
 const makeProductionEvalEngineLayer = (options) => makeEvalEngineLayer().pipe(Layer.provide(Layer.effect(EvalExecutionPort, makeProductionExecutionPort(options))));
 /**

package/dist/test/production-runner.test.js

@@ -216,3 +216,50 @@ test("production runner executes candidate and judge traffic through the live en
         await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
     }
 });
+test("production runner applies the host timeout at node:test instead of a stale request deadline", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-timeout-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "openai/judge" }), minScore: 0.8 });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["openai/cheap"], "openai/judge", ["support-case"]);
+    const gateway = createServer((incoming, outgoing) => {
+        void (async () => {
+            const body = JSON.parse(await readBody(incoming));
+            await new Promise((resolve) => setTimeout(resolve, 25));
+            const content = body.model === "openai/judge"
+                ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
+                : "Helpful answer";
+            outgoing.writeHead(200, { "content-type": "application/json" });
+            outgoing.end(JSON.stringify({
+                model: body.model,
+                choices: [{ message: { role: "assistant", content } }]
+            }));
+        })();
+    });
+    await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve));
+    const address = gateway.address();
+    assert.ok(address !== null && typeof address !== "string");
+    try {
+        const result = await Effect.runPromise(withEvalService({
+            bearerCredential: "parent-only-token",
+            timeoutMs: 15_000
+        }, (service) => service.runComparison({
+            ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
+            candidateModels: ["openai/cheap"],
+            timeoutMs: 5
+        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+        assert.equal(result.models[0]?.cases[0]?.outcome, "passed");
+    }
+    finally {
+        await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.0.11",
+  "version": "1.0.12",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.11",
-    "@velum-labs/routekit-eval-engine": "1.0.11",
-    "@velum-labs/routekit-eval-store": "1.0.11",
-    "@velum-labs/routekit-registry": "1.0.11"
+    "@velum-labs/routekit-eval-contracts": "1.0.12",
+    "@velum-labs/routekit-eval-engine": "1.0.12",
+    "@velum-labs/routekit-eval-store": "1.0.12",
+    "@velum-labs/routekit-registry": "1.0.12"
   },
   "devDependencies": {
     "typescript": "7.0.2"