@velum-labs/routekit-eval-service 1.0.0

package/dist/test/production-runner.test.js

@@ -0,0 +1,218 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient";
+import { Effect, Exit } from "effect";
+import { EvalServiceEstimateError, EvalServiceSpendLimitError } from "../errors.js";
+import { EvalServiceCredentialError, makeRouteKitEvalServiceLayer } from "../production-runner.js";
+import { EvalService } from "../service.js";
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const requestFor = (suitePath, gatewayUrl = "http://127.0.0.1:8080") => ({
+    version: 1,
+    profileId: "support",
+    suitePath,
+    candidateModels: ["openai/cheap", "anthropic/strong"],
+    judgeModel: "openai/judge",
+    gatewayUrl
+});
+const withEvalService = (options, use) => Effect.gen(function* () {
+    return yield* use(yield* EvalService);
+}).pipe(Effect.provide(makeRouteKitEvalServiceLayer({}, options)));
+const readBody = async (request) => {
+    const chunks = [];
+    for await (const chunk of request)
+        chunks.push(Buffer.from(chunk));
+    return Buffer.concat(chunks).toString("utf8");
+};
+const writeManifest = async (root, candidateModels, judgeModel, caseIds) => {
+    await writeFile(path.join(root, "routekit.eval-manifest.json"), `${JSON.stringify({
+        version: 1,
+        profileId: "support",
+        candidateModels,
+        judgeModel,
+        caseCount: caseIds.length,
+        caseIds,
+        maxOutputTokens: 1_024,
+        expectedCallCount: caseIds.length * candidateModels.length * 2
+    })}\n`);
+};
+test("production runner estimates the exact imported nested-loop testdrive suite from its manifest", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-estimate-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    const candidates = ["openai/gpt-5.5", "openai/gpt-5.1", "openai/gpt-4.1-mini"];
+    const caseIds = ["one", "two", "three", "four", "five"];
+    await mkdir(path.join(root, "data"));
+    await writeFile(path.join(root, "data", "cases.json"), `${JSON.stringify(caseIds.map((id) => ({ id, prompt: id })))}\n`);
+    await writeManifest(root, candidates, "openai/gpt-5.5", caseIds);
+    await writeFile(suite, [
+        'import assert from "node:assert/strict";',
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'import cases from "./data/cases.json" with { type: "json" };',
+        'import manifest from "./routekit.eval-manifest.json" with { type: "json" };',
+        "const candidateModels = manifest.candidateModels;",
+        "assert.equal(cases.length, manifest.caseCount);",
+        "const judge = setupJudge({ agent: setupAgent({ model: manifest.judgeModel }) });",
+        "for (const model of candidateModels) {",
+        "  const candidate = setupAgent({ model });",
+        "  for (const testCase of cases) {",
+        "    test(`${model} / ${testCase.id}`, async () => {",
+        "      const run = await candidate.run({ prompt: testCase.prompt, caseId: testCase.id });",
+        "      run.toComplete();",
+        '      await judge.autoEvals({ criteria: "correct", prompt: testCase.prompt, run });',
+        "    });",
+        "  }",
+        "}"
+    ].join("\n"));
+    const result = await Effect.runPromise(withEvalService({}, (service) => Effect.gen(function* () {
+        yield* service.validate(suite);
+        return yield* service.estimate({
+            ...requestFor(suite),
+            candidateModels: candidates,
+            judgeModel: "openai/gpt-5.5"
+        }, "pilot");
+    })).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+    assert.equal(result.callCount, 30);
+    assert.equal(result.pricingKnown, true);
+    assert.equal((result.maximumCostUsd ?? 0) > 0, true);
+});
+test("production runner rejects a manifest for a different profile", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-profile-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, 'import { test } from "node:test"; test("case", () => {});\n');
+    await writeManifest(root, ["openai/cheap", "anthropic/strong"], "openai/judge", ["case"]);
+    const manifestPath = path.join(root, "routekit.eval-manifest.json");
+    const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+    await writeFile(manifestPath, `${JSON.stringify({ ...manifest, profileId: "different-profile" })}\n`);
+    const exit = await Effect.runPromise(withEvalService({}, (service) => service.estimate(requestFor(suite), "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+    assert.equal(Exit.isFailure(exit), true);
+    if (Exit.isFailure(exit)) {
+        assert.match(String(exit.cause), /profile or models do not match/u);
+        assert.match(String(exit.cause), new RegExp(EvalServiceEstimateError.name));
+    }
+});
+test("production runner fails paid execution clearly when no credential was injected", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-token-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, 'import { test } from "node:test"; test("case", () => {});\n');
+    await writeManifest(root, ["openai/cheap", "anthropic/strong"], "openai/judge", ["case"]);
+    const exit = await Effect.runPromise(withEvalService({}, (service) => service.runComparison(requestFor(suite), "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+    assert.equal(Exit.isFailure(exit), true);
+    if (Exit.isFailure(exit)) {
+        assert.match(String(exit.cause), /requires an injected bearer credential/u);
+        assert.match(String(exit.cause), new RegExp(EvalServiceCredentialError.name));
+    }
+});
+test("production runner enforces spendLimitUsd before live execution", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-spend-limit-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "openai/gpt-5.5" }) });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "openai/gpt-5.1" }).run("Help");',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["openai/gpt-5.1"], "openai/gpt-5.5", ["support-case"]);
+    const exit = await Effect.runPromise(withEvalService({ bearerCredential: "unused-token" }, (service) => service.runComparison({
+        ...requestFor(suite),
+        candidateModels: ["openai/gpt-5.1"],
+        judgeModel: "openai/gpt-5.5",
+        spendLimitUsd: 0
+    }, "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+    assert.equal(Exit.isFailure(exit), true);
+    if (Exit.isFailure(exit)) {
+        assert.match(String(exit.cause), /exceeds spendLimitUsd/u);
+        assert.match(String(exit.cause), new RegExp(EvalServiceSpendLimitError.name));
+    }
+});
+test("production runner fails closed when a spend limit has unknown pricing", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-spend-unknown-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "unknown/judge" }) });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "unknown/candidate" }).run("Help");',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["unknown/candidate"], "unknown/judge", ["support-case"]);
+    const exit = await Effect.runPromise(withEvalService({ bearerCredential: "unused-token" }, (service) => service.runComparison({
+        ...requestFor(suite),
+        candidateModels: ["unknown/candidate"],
+        judgeModel: "unknown/judge",
+        spendLimitUsd: 100
+    }, "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
+    assert.equal(Exit.isFailure(exit), true);
+    if (Exit.isFailure(exit)) {
+        assert.match(String(exit.cause), /pricing is unknown/u);
+        assert.match(String(exit.cause), new RegExp(EvalServiceSpendLimitError.name));
+    }
+});
+test("production runner executes candidate and judge traffic through the live engine", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-live-"));
+    roots.push(root);
+    const suite = path.join(root, "support.eval.ts");
+    await writeFile(suite, [
+        'import { test } from "node:test";',
+        'import { setupAgent, setupJudge } from "routekit/eval";',
+        'const judge = setupJudge({ agent: setupAgent({ model: "openai/judge" }), minScore: 0.8 });',
+        'test("support case", async () => {',
+        '  const run = await setupAgent({ model: "openai/cheap" }).run({ prompt: "Help", caseId: "support-case" });',
+        "  run.toComplete();",
+        '  await judge.autoEvals({ criteria: "Helpful", prompt: "Help", run });',
+        "});"
+    ].join("\n"));
+    await writeManifest(root, ["openai/cheap"], "openai/judge", ["support-case"]);
+    const calls = [];
+    const gateway = createServer((incoming, outgoing) => {
+        void (async () => {
+            const body = JSON.parse(await readBody(incoming));
+            calls.push({
+                authorization: incoming.headers.authorization,
+                maxOutputTokens: body.max_completion_tokens,
+                model: body.model
+            });
+            const content = body.model === "openai/judge"
+                ? JSON.stringify({ pass: true, reason: "helpful", score: 0.9 })
+                : "Helpful answer";
+            outgoing.writeHead(200, { "content-type": "application/json" });
+            outgoing.end(JSON.stringify({
+                model: body.model,
+                choices: [{ message: { role: "assistant", content } }]
+            }));
+        })();
+    });
+    await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve));
+    const address = gateway.address();
+    assert.ok(address !== null && typeof address !== "string");
+    try {
+        const result = await Effect.runPromise(withEvalService({ bearerCredential: "parent-only-token" }, (service) => service.runComparison({
+            ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
+            candidateModels: ["openai/cheap"]
+        }, "pilot")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+        assert.deepEqual(calls.map(({ model }) => model), ["openai/cheap", "openai/judge"]);
+        assert.equal(calls.every(({ authorization }) => authorization === "Bearer parent-only-token"), true);
+        assert.equal(calls.every(({ maxOutputTokens }) => maxOutputTokens === 1_024), true);
+        assert.deepEqual(result.models.map(({ model }) => model), ["openai/cheap"]);
+        assert.equal(JSON.stringify(result).includes("parent-only-token"), false);
+    }
+    finally {
+        await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));
+    }
+});

package/dist/test/service.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/service.test.js

@@ -0,0 +1,162 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { EvalEngine } from "@velum-labs/routekit-eval-engine";
+import { Effect, Fiber, Layer } from "effect";
+import { EvalService, makeEvalServiceLayer } from "../service.js";
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const dimensions = [
+    "gateway-protocol",
+    "eval-routing",
+    "account-pooling",
+    "typescript-maintenance",
+    "release-operations"
+];
+const basis = {
+    version: 2,
+    basisDigest: "matrix-definition-set",
+    dimensions: dimensions.map((id) => ({
+        id,
+        description: `Requests about ${id}`,
+        includes: [`Tasks specifically involving ${id}`],
+        excludes: [`Tasks unrelated to ${id}`]
+    }))
+};
+const suites = (root) => dimensions.map((dimensionId) => ({
+    dimensionId,
+    suitePath: path.join(root, dimensionId, `${dimensionId}.eval.ts`)
+}));
+const prepareSuites = async (root, caseIds = ["case-1"]) => {
+    await Promise.all(dimensions.map(async (dimensionId) => {
+        const directory = path.join(root, dimensionId);
+        await mkdir(directory, { recursive: true });
+        await writeFile(path.join(directory, `${dimensionId}.eval.ts`), 'import { test } from "node:test"; test("case", () => {});\n');
+        await writeFile(path.join(directory, "routekit.eval-manifest.json"), `${JSON.stringify({
+            version: 1,
+            profileId: dimensionId,
+            candidateModels: ["openai/cheap", "anthropic/strong"],
+            judgeModel: "openai/judge",
+            caseCount: caseIds.length,
+            caseIds,
+            maxOutputTokens: 256,
+            expectedCallCount: caseIds.length * 4
+        })}\n`);
+    }));
+};
+const mockEngineLayer = (runComparison, onValidate) => Layer.succeed(EvalEngine, EvalEngine.of({
+    discover: (target) => Effect.succeed({
+        searchRoot: target,
+        workingDirectory: path.dirname(target),
+        files: [target]
+    }),
+    validate: (target) => {
+        const dimensionId = path.basename(target, ".eval.ts");
+        onValidate?.(dimensionId);
+        return Effect.succeed({
+            searchRoot: target,
+            workingDirectory: path.dirname(target),
+            files: [target],
+            suiteDigest: `suite-${dimensionId}`
+        });
+    },
+    runComparison
+}));
+const resultFor = (request) => ({
+    version: 1,
+    comparisonId: `comparison-${request.profileId}`,
+    profileId: request.profileId,
+    suiteDigest: `suite-${request.profileId}`,
+    judgeModel: request.judgeModel,
+    startedAt: "2026-08-18T00:00:00.000Z",
+    finishedAt: "2026-08-18T00:01:00.000Z",
+    models: request.candidateModels.map((model) => ({
+        model,
+        cases: [
+            {
+                caseId: "case-1",
+                outcome: "passed",
+                measurement: {
+                    judgeScore: model === "openai/cheap" ? 0.9 : 0.95,
+                    costUsd: model === "openai/cheap" ? 0.001 : 0.1,
+                    durationMs: model === "openai/cheap" ? 200 : 100
+                }
+            }
+        ]
+    }))
+});
+const qualification = (root) => ({
+    basis,
+    candidateModels: ["openai/cheap", "anthropic/strong"],
+    classifierModel: "openai/classifier",
+    judgeModel: "openai/judge",
+    objective: { kind: "highest-quality" },
+    maximumUnknownWeight: 0.2,
+    suites: suites(root)
+});
+test("dimension matrix qualification inspects every manifest before publishing one activation", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-dimension-matrix-"));
+    roots.push(root);
+    await prepareSuites(root);
+    const events = [];
+    const snapshotRoot = path.join(root, "snapshots");
+    const engine = mockEngineLayer((request) => Effect.sync(() => {
+        events.push(`run:${request.profileId}`);
+        return resultFor(request);
+    }), (dimensionId) => events.push(`inspect:${dimensionId}`));
+    const layer = makeEvalServiceLayer({
+        gatewayUrl: "http://127.0.0.1:8080/v1",
+        snapshotRoot
+    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    const result = await Effect.runPromise(Effect.gen(function* () {
+        return yield* (yield* EvalService).qualifyDimensionMatrix(qualification(root));
+    }).pipe(Effect.provide(layer)));
+    assert.equal(result.comparisons.length, dimensions.length);
+    assert.equal(result.snapshot.evidence.length, dimensions.length * 2);
+    assert.deepEqual(events, [
+        ...dimensions.map((id) => `inspect:${id}`),
+        ...dimensions.map((id) => `run:${id}`)
+    ]);
+    const persisted = JSON.parse(await readFile(path.join(snapshotRoot, "published-routing.json"), "utf8"));
+    assert.equal(persisted.evidenceDigest, result.snapshot.evidenceDigest);
+});
+test("dimension matrix qualification never publishes incomplete comparison evidence", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-dimension-matrix-incomplete-"));
+    roots.push(root);
+    await prepareSuites(root, ["case-1", "case-2"]);
+    const snapshotRoot = path.join(root, "snapshots");
+    const engine = mockEngineLayer((request) => Effect.succeed(resultFor(request)));
+    const layer = makeEvalServiceLayer({
+        gatewayUrl: "http://127.0.0.1:8080/v1",
+        snapshotRoot
+    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    const exit = await Effect.runPromiseExit(Effect.gen(function* () {
+        return yield* (yield* EvalService).qualifyDimensionMatrix(qualification(root));
+    }).pipe(Effect.provide(layer)));
+    assert.equal(exit._tag, "Failure");
+    await assert.rejects(readFile(path.join(snapshotRoot, "published-routing.json"), "utf8"), /ENOENT/u);
+});
+test("interrupted qualification leaves no half-published activation for a daemon restart", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-dimension-matrix-interrupted-"));
+    roots.push(root);
+    await prepareSuites(root);
+    const snapshotRoot = path.join(root, "snapshots");
+    let runs = 0;
+    const engine = mockEngineLayer((request) => {
+        runs += 1;
+        return runs === 1 ? Effect.succeed(resultFor(request)) : Effect.never;
+    });
+    const layer = makeEvalServiceLayer({
+        gatewayUrl: "http://127.0.0.1:8080/v1",
+        snapshotRoot
+    }).pipe(Layer.provide(engine), Layer.provide(NodeServicesLayer));
+    await Effect.runPromise(Effect.gen(function* () {
+        const fiber = yield* Effect.forkChild((yield* EvalService).qualifyDimensionMatrix(qualification(root)));
+        yield* Effect.yieldNow;
+        yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(layer)));
+    await assert.rejects(readFile(path.join(snapshotRoot, "published-routing.json"), "utf8"), /ENOENT/u);
+});

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@velum-labs/routekit-eval-service",
+  "private": false,
+  "version": "1.0.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/velum-labs/routekit.git",
+    "directory": "packages/eval-service"
+  },
+  "description": "Effect composition for RouteKit eval comparisons and routing policy publication.",
+  "license": "Apache-2.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./effect": {
+      "types": "./dist/effect-api.d.ts",
+      "default": "./dist/effect-api.js"
+    }
+  },
+  "files": [
+    "dist",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public",
+    "provenance": true
+  },
+  "dependencies": {
+    "@effect/platform-node": "4.0.0-rc.108",
+    "effect": "4.0.0-rc.108",
+    "@velum-labs/routekit-eval-contracts": "1.0.0",
+    "@velum-labs/routekit-registry": "1.0.0",
+    "@velum-labs/routekit-eval-engine": "1.0.0",
+    "@velum-labs/routekit-eval-store": "1.0.0"
+  },
+  "devDependencies": {
+    "typescript": "7.0.2"
+  },
+  "keywords": [
+    "routekit",
+    "eval",
+    "routing",
+    "effect"
+  ],
+  "scripts": {
+    "build": "tsc -b",
+    "clean": "tsc -b --clean",
+    "test": "node --test \"dist/test/*.test.js\""
+  }
+}