@velum-labs/routekit-eval-service 1.0.11 → 1.0.13

package/dist/layer-options.d.ts

@@ -7,4 +7,20 @@ export type RouteKitEvalServiceOptions = Omit<NodeTestExecutionOptions, "bridgeO
      * execution fails before starting a child when it is absent.
      */
     readonly bearerCredential?: string;
+    /**
+     * Host-owned per-test deadline.
+     *
+     * When set, the production execution adapter applies this value at the
+     * node:test boundary even if an intermediate comparison request carries a
+     * stale or missing timeout.
+     */
+    readonly timeoutMs?: number;
+    /**
+     * Run the node:test child from a temporary copy of the reviewed suite.
+     *
+     * Qualification enables this so a project-local `routekit/eval` or `ori/eval`
+     * package cannot shadow the bundled bridge-aware SDK used by the production
+     * execution adapter.
+     */
+    readonly isolateExecutionFromProjectSdk?: boolean;
 };

package/dist/production-runner.js

@@ -1,6 +1,6 @@
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { EvalEngineExecutionError, EvalExecutionPort, makeEvalEngineLayer, makeRouteKitEvalExecutionPortService } from "@velum-labs/routekit-eval-engine";
-import { Data, Effect, Layer, Stream } from "effect";
+import { Data, Effect, FileSystem, Layer, Path, Stream } from "effect";
 import { HttpClient } from "effect/unstable/http";
 import { makeEvalServiceLayer } from "./service.js";
 export class EvalServiceCredentialError extends Data.TaggedError("EvalServiceCredentialError") {
@@ -8,25 +8,61 @@ export class EvalServiceCredentialError extends Data.TaggedError("EvalServiceCre
         return this.detail;
     }
 }
-const makeProductionExecutionPort = (options) => Effect.map(HttpClient.HttpClient, (httpClient) => {
+const makeProductionExecutionPort = (options) => Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
     const bearerCredential = options.bearerCredential?.trim();
     if (bearerCredential === undefined || bearerCredential.length === 0) {
-        return {
+        return EvalExecutionPort.of({
             execute: () => Stream.fail(new EvalEngineExecutionError({
                 cause: new EvalServiceCredentialError({
                     detail: "RouteKit Eval comparison execution requires an injected bearer credential."
                 }),
                 detail: "RouteKit Eval comparison execution requires an injected bearer credential."
             }))
-        };
+        });
     }
-    return makeRouteKitEvalExecutionPortService({
+    const httpClient = yield* HttpClient.HttpClient;
+    const paths = yield* Path.Path;
+    const execution = makeRouteKitEvalExecutionPortService({
         bearerCredential,
         ...(options.childEnvironment === undefined
             ? {}
             : { childEnvironment: options.childEnvironment }),
         ...(options.execPath === undefined ? {} : { execPath: options.execPath })
     }, httpClient);
+    return EvalExecutionPort.of({
+        execute: (input) => {
+            const deadlineInput = options.timeoutMs === undefined
+                ? input
+                : {
+                    ...input,
+                    request: {
+                        ...input.request,
+                        timeoutMs: options.timeoutMs
+                    }
+                };
+            if (options.isolateExecutionFromProjectSdk !== true) {
+                return execution.execute(deadlineInput);
+            }
+            return Stream.unwrap(Effect.gen(function* () {
+                const root = yield* Effect.acquireRelease(fs.makeTempDirectory({ prefix: "routekit-eval-execution-" }), (directory) => fs.remove(directory, { recursive: true }).pipe(Effect.ignore));
+                const workingDirectory = paths.join(root, "suite");
+                yield* fs.copy(input.discovery.workingDirectory, workingDirectory);
+                const discovery = {
+                    ...input.discovery,
+                    workingDirectory,
+                    files: input.discovery.files.map((file) => paths.join(workingDirectory, paths.relative(input.discovery.workingDirectory, file)))
+                };
+                return execution.execute({
+                    ...deadlineInput,
+                    discovery
+                });
+            }).pipe(Effect.mapError((cause) => new EvalEngineExecutionError({
+                cause,
+                detail: "RouteKit Eval could not isolate the reviewed suite from project-local SDK packages."
+            })))).pipe(Stream.scoped);
+        }
+    });
 });
 const makeProductionEvalEngineLayer = (options) => makeEvalEngineLayer().pipe(Layer.provide(Layer.effect(EvalExecutionPort, makeProductionExecutionPort(options))));
 /**

package/dist/test/production-runner.test.js

@@ -40,6 +40,27 @@ const writeManifest = async (root, candidateModels, judgeModel, caseIds) => {
         expectedCallCount: caseIds.length * candidateModels.length * 2
     })}\n`);
 };
+const writeShadowEvalSdk = async (root, hiddenTimeoutMs) => {
+    for (const name of ["routekit", "ori"]) {
+        const directory = path.join(root, "node_modules", name);
+        await mkdir(directory, { recursive: true });
+        await writeFile(path.join(directory, "package.json"), `${JSON.stringify({
+            name,
+            private: true,
+            type: "module",
+            exports: { "./eval": "./eval.js" }
+        })}\n`);
+        await writeFile(path.join(directory, "eval.js"), [
+            "export const setupAgent = () => ({",
+            "  run: async () => {",
+            `    await new Promise((resolve) => setTimeout(resolve, ${String(hiddenTimeoutMs)}));`,
+            '    throw new Error("project SDK hidden timeout");',
+            "  }",
+            "});",
+            "export const setupJudge = () => ({ autoEvals: async () => {} });"
+        ].join("\n"));
+    }
+};
 test("production runner estimates the exact imported nested-loop testdrive suite from its manifest", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-estimate-"));
     roots.push(root);
@@ -216,3 +237,116 @@ test("production runner executes candidate and judge traffic through the live en
         await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
     }
 });
+test("production runner applies the host timeout at node:test instead of a stale request deadline", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-timeout-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "openai/judge" }), minScore: 0.8 });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["openai/cheap"], "openai/judge", ["support-case"]);
+    const gateway = createServer((incoming, outgoing) => {
+        void (async () => {
+            const body = JSON.parse(await readBody(incoming));
+            await new Promise((resolve) => setTimeout(resolve, 25));
+            const content = body.model === "openai/judge"
+                ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
+                : "Helpful answer";
+            outgoing.writeHead(200, { "content-type": "application/json" });
+            outgoing.end(JSON.stringify({
+                model: body.model,
+                choices: [{ message: { role: "assistant", content } }]
+            }));
+        })();
+    });
+    await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve));
+    const address = gateway.address();
+    assert.ok(address !== null && typeof address !== "string");
+    try {
+        const result = await Effect.runPromise(withEvalService({
+            bearerCredential: "parent-only-token",
+            timeoutMs: 15_000
+        }, (service) => service.runComparison({
+            ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
+            candidateModels: ["openai/cheap"],
+            timeoutMs: 5
+        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+        assert.equal(result.models[0]?.cases[0]?.outcome, "passed");
+    }
+    finally {
+        await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
+    }
+});
+test("production runner isolates qualification from a project SDK with a shorter hidden abort", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-isolated-sdk-"));
+    roots.push(root);
+    await writeShadowEvalSdk(root, 20);
+    const suiteRoot = path.join(root, ".routekit", "evals", "plans", "plan", "dimensions", "support");
+    await mkdir(suiteRoot, { recursive: true });
+    const suite = path.join(suiteRoot, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "openai/judge" }), minScore: 0.8 });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(suiteRoot, ["openai/cheap"], "openai/judge", ["support-case"]);
+    const calls = [];
+    const gateway = createServer((incoming, outgoing) => {
+        void (async () => {
+            const body = JSON.parse(await readBody(incoming));
+            calls.push(String(body.model));
+            await new Promise((resolve) => setTimeout(resolve, 75));
+            const content = body.model === "openai/judge"
+                ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
+                : "Helpful answer";
+            outgoing.writeHead(200, { "content-type": "application/json" });
+            outgoing.end(JSON.stringify({
+                model: body.model,
+                choices: [{ message: { role: "assistant", content } }]
+            }));
+        })();
+    });
+    await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve));
+    const address = gateway.address();
+    assert.ok(address !== null && typeof address !== "string");
+    try {
+        const shadowed = await Effect.runPromise(withEvalService({
+            bearerCredential: "parent-only-token",
+            timeoutMs: 15_000
+        }, (service) => service.runComparison({
+            ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
+            candidateModels: ["openai/cheap"],
+            timeoutMs: 20
+        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+        assert.equal(Exit.isFailure(shadowed), true);
+        assert.deepEqual(calls, []);
+        const started = Date.now();
+        const result = await Effect.runPromise(withEvalService({
+            bearerCredential: "parent-only-token",
+            isolateExecutionFromProjectSdk: true,
+            timeoutMs: 15_000
+        }, (service) => service.runComparison({
+            ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
+            candidateModels: ["openai/cheap"],
+            timeoutMs: 20
+        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+        assert.deepEqual(calls, ["openai/cheap", "openai/judge"]);
+        assert.equal(result.models[0]?.cases[0]?.outcome, "passed");
+        assert.ok(Date.now() - started >= 75, "the project SDK's 20ms abort must not bound execution");
+    }
+    finally {
+        await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.0.11",
+  "version": "1.0.13",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.11",
-    "@velum-labs/routekit-eval-engine": "1.0.11",
-    "@velum-labs/routekit-eval-store": "1.0.11",
-    "@velum-labs/routekit-registry": "1.0.11"
+    "@velum-labs/routekit-eval-contracts": "1.0.13",
+    "@velum-labs/routekit-eval-engine": "1.0.13",
+    "@velum-labs/routekit-eval-store": "1.0.13",
+    "@velum-labs/routekit-registry": "1.0.13"
   },
   "devDependencies": {
     "typescript": "7.0.2"