@velum-labs/routekit-eval-service 1.0.16 → 1.0.18

package/dist/effect-api.d.ts

@@ -1,7 +1,7 @@
 export type { CompileDimensionEvidenceMatrixInput, CompiledDimensionEvidenceMatrix, DimensionComparisonEvidenceInput } from "./dimension-evidence.js";
 export { compileDimensionEvidenceMatrix, DimensionEvidenceCompilationError, wilsonLowerBound95 } from "./dimension-evidence.js";
 export { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
-export type { RouteKitEvalServiceOptions } from "./production-runner.js";
+export type { RouteKitEvalGatewayCallEvent, RouteKitEvalServiceOptions } from "./production-runner.js";
 export { EvalServiceCredentialError, makeRouteKitEvalServiceLayer } from "./production-runner.js";
 export type { DimensionMatrixQualificationInput, DimensionMatrixQualificationResult, DimensionMatrixSuite, EvalComparisonEstimate, EvalComparisonMode, EvalRunConfiguration, EvalServiceConfiguration, EvalServiceError, EvalServiceShape, EvalSuiteInspection } from "./service.js";
 export { EvalService } from "./service.js";

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 export type { CompileDimensionEvidenceMatrixInput, CompiledDimensionEvidenceMatrix, DimensionComparisonEvidenceInput } from "./dimension-evidence.js";
 export { compileDimensionEvidenceMatrix, DimensionEvidenceCompilationError, wilsonLowerBound95 } from "./dimension-evidence.js";
 export { EvalServiceComparisonError, EvalServiceConfigurationError, EvalServiceEstimateError, EvalServicePolicyError, EvalServicePublicationError, EvalServiceSpendLimitError, EvalServiceValidationError } from "./errors.js";
-export type { RouteKitEvalServiceOptions } from "./production-runner.js";
+export type { RouteKitEvalGatewayCallEvent, RouteKitEvalServiceOptions } from "./production-runner.js";
 export { EvalServiceCredentialError, makeRouteKitEvalServiceLayer } from "./production-runner.js";
 export type { DimensionMatrixQualificationInput, DimensionMatrixQualificationResult, DimensionMatrixSuite, EvalComparisonEstimate, EvalComparisonMode, EvalRunConfiguration, EvalServiceConfiguration, EvalServiceError, EvalServiceShape, EvalSuiteInspection } from "./service.js";
 export { EvalService } from "./service.js";

package/dist/layer-options.d.ts

@@ -1,4 +1,15 @@
 import type { NodeTestExecutionOptions } from "@velum-labs/routekit-eval-engine";
+import type { Effect } from "effect";
+export type RouteKitEvalGatewayCallEvent = {
+    readonly observationId: string;
+    readonly phase: "issued";
+    readonly role: "candidate" | "judge";
+} | {
+    readonly callId?: string;
+    readonly observationId: string;
+    readonly phase: "completed";
+    readonly role: "candidate" | "judge";
+};
 export type RouteKitEvalServiceOptions = Omit<NodeTestExecutionOptions, "bridgeOrigin"> & {
     /**
      * Injected RouteKit data-plane credential.
@@ -15,6 +26,12 @@ export type RouteKitEvalServiceOptions = Omit<NodeTestExecutionOptions, "bridgeO
      * stale or missing timeout.
      */
     readonly timeoutMs?: number;
+    /**
+     * Observe the exact parent-owned HTTP client used by the child gateway
+     * bridge. The callback runs when an attributed request is issued and again
+     * when a model call id is available from its response.
+     */
+    readonly observeGatewayCall?: (event: RouteKitEvalGatewayCallEvent) => Effect.Effect<void>;
     /**
      * Run the node:test child from a temporary copy of the reviewed suite.
      *

package/dist/production-runner.d.ts

@@ -1,8 +1,8 @@
 import { Layer } from "effect";
 import { HttpClient } from "effect/unstable/http";
-import type { RouteKitEvalServiceOptions } from "./layer-options.js";
+import type { RouteKitEvalGatewayCallEvent, RouteKitEvalServiceOptions } from "./layer-options.js";
 import { EvalService, type EvalServiceConfiguration } from "./service.js";
-export type { RouteKitEvalServiceOptions };
+export type { RouteKitEvalGatewayCallEvent, RouteKitEvalServiceOptions };
 declare const EvalServiceCredentialError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
     readonly _tag: "EvalServiceCredentialError";
 } & Readonly<A>;

package/dist/production-runner.js

@@ -1,4 +1,5 @@
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { EVAL_ATTRIBUTION_HEADER } from "@velum-labs/routekit-eval-contracts";
 import { EvalEngineExecutionError, EvalExecutionPort, makeEvalEngineLayer, makeRouteKitEvalExecutionPortService } from "@velum-labs/routekit-eval-engine";
 import { Data, Effect, FileSystem, Layer, Path, Stream } from "effect";
 import { HttpClient } from "effect/unstable/http";
@@ -8,11 +9,46 @@ export class EvalServiceCredentialError extends Data.TaggedError("EvalServiceCre
         return this.detail;
     }
 }
+const MODEL_CALL_ID_HEADER = "x-routekit-model-call-id";
+const gatewayCallRole = (raw) => {
+    if (raw === undefined)
+        return undefined;
+    try {
+        const decoded = JSON.parse(raw);
+        return decoded.role === "candidate" || decoded.role === "judge" ? decoded.role : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+const observeGatewayCalls = (client, observe) => {
+    let nextObservationId = 0;
+    return HttpClient.transform(client, (responseEffect, request) => {
+        const role = gatewayCallRole(request.headers[EVAL_ATTRIBUTION_HEADER]);
+        if (role === undefined)
+            return responseEffect;
+        return Effect.gen(function* () {
+            nextObservationId += 1;
+            const observationId = `gateway-call-${String(nextObservationId)}`;
+            yield* observe({ observationId, phase: "issued", role });
+            const response = yield* responseEffect;
+            const rawCallId = response.headers[MODEL_CALL_ID_HEADER]?.trim();
+            yield* observe({
+                observationId,
+                phase: "completed",
+                role,
+                ...(rawCallId === undefined || rawCallId.length === 0 ? {} : { callId: rawCallId })
+            });
+            return response;
+        });
+    });
+};
 const makeProductionExecutionPort = (options) => Effect.gen(function* () {
     const fs = yield* FileSystem.FileSystem;
     const bearerCredential = options.bearerCredential?.trim();
     if (bearerCredential === undefined || bearerCredential.length === 0) {
         return EvalExecutionPort.of({
+            nodeTestExecPath: options.execPath ?? globalThis.process.execPath,
             execute: () => Stream.fail(new EvalEngineExecutionError({
                 cause: new EvalServiceCredentialError({
                     detail: "RouteKit Eval comparison execution requires an injected bearer credential."
@@ -21,7 +57,10 @@ const makeProductionExecutionPort = (options) => Effect.gen(function* () {
             }))
         });
     }
-    const httpClient = yield* HttpClient.HttpClient;
+    const baseHttpClient = yield* HttpClient.HttpClient;
+    const httpClient = options.observeGatewayCall === undefined
+        ? baseHttpClient
+        : observeGatewayCalls(baseHttpClient, options.observeGatewayCall);
     const paths = yield* Path.Path;
     const execution = makeRouteKitEvalExecutionPortService({
         bearerCredential,
@@ -31,6 +70,7 @@ const makeProductionExecutionPort = (options) => Effect.gen(function* () {
         ...(options.execPath === undefined ? {} : { execPath: options.execPath })
     }, httpClient);
     return EvalExecutionPort.of({
+        nodeTestExecPath: execution.nodeTestExecPath,
         execute: (input) => {
             const deadlineInput = options.timeoutMs === undefined
                 ? input

package/dist/test/production-runner.test.js

@@ -201,6 +201,7 @@ test("production runner executes candidate and judge traffic through the live en
     ].join("\n"));
     await writeManifest(root, ["openai/cheap"], "openai/judge", ["support-case"]);
     const calls = [];
+    const observed = [];
     const gateway = createServer((incoming, outgoing) => {
         void (async () => {
             const body = JSON.parse(await readBody(incoming));
@@ -212,7 +213,10 @@ test("production runner executes candidate and judge traffic through the live en
             const content = body.model === "openai/judge"
                 ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
                 : "Helpful answer";
-            outgoing.writeHead(200, { "content-type": "application/json" });
+            outgoing.writeHead(200, {
+                "content-type": "application/json",
+                "x-routekit-model-call-id": `model-call-${String(calls.length)}`
+            });
             outgoing.end(JSON.stringify({
                 model: body.model,
                 choices: [{ message: { role: "assistant", content } }]
@@ -223,13 +227,38 @@ test("production runner executes candidate and judge traffic through the live en
     const address = gateway.address();
     assert.ok(address !== null && typeof address !== "string");
     try {
-        const result = await Effect.runPromise(withEvalService({ bearerCredential: "parent-only-token" }, (service) => service.runComparison({
+        const result = await Effect.runPromise(withEvalService({
+            bearerCredential: "parent-only-token",
+            observeGatewayCall: (event) => Effect.sync(() => {
+                observed.push({
+                    phase: event.phase,
+                    role: event.role,
+                    ...(event.phase === "completed" && event.callId !== undefined
+                        ? { callId: event.callId }
+                        : {})
+                });
+            })
+        }, (service) => service.runComparison({
             ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
             candidateModels: ["openai/cheap"]
         }, "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
         assert.deepEqual(calls.map(({ model }) => model), ["openai/cheap", "openai/judge"]);
         assert.equal(calls.every(({ authorization }) => authorization === "Bearer parent-only-token"), true);
         assert.equal(calls.every(({ maxOutputTokens }) => maxOutputTokens === 1_024), true);
+        assert.deepEqual(observed, [
+            { phase: "issued", role: "candidate" },
+            {
+                callId: "model-call-1",
+                phase: "completed",
+                role: "candidate"
+            },
+            { phase: "issued", role: "judge" },
+            {
+                callId: "model-call-2",
+                phase: "completed",
+                role: "judge"
+            }
+        ]);
         assert.deepEqual(result.models.map(({ model }) => model), ["openai/cheap"]);
         assert.equal(JSON.stringify(result).includes("parent-only-token"), false);
     }
@@ -284,6 +313,38 @@ test("production runner applies the host timeout at node:test instead of a stale
         await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
     }
 });
+test("production runner uses its selected Node runtime for validation before observation", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-spawn-failure-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent } from "routekit/eval";',
+        'test("support case", async () => {',
+        '  await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["openai/cheap"], "openai/judge", ["support-case"]);
+    const observed = [];
+    const exit = await Effect.runPromise(withEvalService({
+        bearerCredential: "parent-only-token",
+        execPath: path.join(root, "missing-node"),
+        observeGatewayCall: (event) => Effect.sync(() => {
+            observed.push(event);
+        }),
+        timeoutMs: 600_000
+    }, (service) => service.runComparison({
+        ...requestFor(suite),
+        candidateModels: ["openai/cheap"]
+    }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+    assert.equal(Exit.isFailure(exit), true);
+    assert.deepEqual(observed, []);
+    if (Exit.isFailure(exit)) {
+        assert.match(String(exit.cause), /validation failed/iu);
+        assert.match(String(exit.cause), /could not be loaded safely through node:test/iu);
+        assert.match(String(exit.cause), /missing-node/iu);
+    }
+});
 test("production runner isolates qualification from a project SDK with a shorter hidden abort", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-isolated-sdk-"));
     roots.push(root);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.0.16",
+  "version": "1.0.18",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.16",
-    "@velum-labs/routekit-eval-engine": "1.0.16",
-    "@velum-labs/routekit-eval-store": "1.0.16",
-    "@velum-labs/routekit-registry": "1.0.16"
+    "@velum-labs/routekit-eval-contracts": "1.0.18",
+    "@velum-labs/routekit-eval-engine": "1.0.18",
+    "@velum-labs/routekit-eval-store": "1.0.18",
+    "@velum-labs/routekit-registry": "1.0.18"
   },
   "devDependencies": {
     "typescript": "7.0.2"