@velum-labs/routekit-eval-service 1.0.24 → 1.0.26

package/dist/dimension-evidence.js

@@ -23,36 +23,40 @@ export function compileDimensionEvidenceMatrix(input) {
         }
         comparisons.set(dimension.dimensionId, dimension);
     }
-    for (const dimension of input.basis.dimensions) {
-        if (!comparisons.has(dimension.id)) {
-            fail(`comparison is missing dimension ${JSON.stringify(dimension.id)}`);
-        }
+    if (comparisons.size === 0) {
+        fail("comparison is missing every selected dimension");
     }
     const evidence = [];
     for (const dimension of input.basis.dimensions) {
         const dimensionInput = comparisons.get(dimension.id);
+        if (dimensionInput === undefined) {
+            continue;
+        }
         evidence.push(...compileDimension(dimensionInput, candidateModels));
     }
     evidence.sort((left, right) => left.model.localeCompare(right.model) || left.dimensionId.localeCompare(right.dimensionId));
     const evidenceDigest = stableDigest(evidence);
     // Reuse the public contract's exhaustive matrix validation before returning
     // an artifact that can be handed to the v2 snapshot store.
-    try {
-        assertPublishedRoutingActivation({
-            version: COMPOSITIONAL_ROUTING_VERSION,
-            generatedAt: "1970-01-01T00:00:00.000Z",
-            basisDigest: input.basis.basisDigest,
-            evidenceDigest,
-            classifierModel: "routekit/evidence-validator",
-            objective: { kind: "highest-quality" },
-            maximumUnknownWeight: 1,
-            dimensions: [...input.basis.dimensions],
-            candidateModels: [...candidateModels],
-            evidence
-        });
-    }
-    catch (cause) {
-        fail(`compiled model-dimension evidence is invalid: ${detailOf(cause)}`);
+    const compiledDimensions = input.basis.dimensions.filter((dimension) => comparisons.has(dimension.id));
+    if (compiledDimensions.length === input.basis.dimensions.length) {
+        try {
+            assertPublishedRoutingActivation({
+                version: COMPOSITIONAL_ROUTING_VERSION,
+                generatedAt: "1970-01-01T00:00:00.000Z",
+                basisDigest: input.basis.basisDigest,
+                evidenceDigest,
+                classifierModel: "routekit/evidence-validator",
+                objective: { kind: "highest-quality" },
+                maximumUnknownWeight: 1,
+                dimensions: [...input.basis.dimensions],
+                candidateModels: [...candidateModels],
+                evidence
+            });
+        }
+        catch (cause) {
+            fail(`compiled model-dimension evidence is invalid: ${detailOf(cause)}`);
+        }
     }
     return { evidenceDigest, evidence };
 }

package/dist/test/dimension-evidence.test.js

@@ -109,7 +109,8 @@ test("partial duration measurements do not produce a misleading percentile", ()
 test("matrix compilation rejects missing and duplicate dimensions or candidates", () => {
     const missingDimension = input();
     missingDimension.comparisons.pop();
-    assert.throws(() => compileDimensionEvidenceMatrix(missingDimension), /missing dimension/);
+    const partial = compileDimensionEvidenceMatrix(missingDimension);
+    assert.equal(partial.evidence.length, (dimensionIds.length - 1) * models.length);
     const duplicateDimension = input();
     duplicateDimension.comparisons[1] = duplicateDimension.comparisons[0];
     assert.throws(() => compileDimensionEvidenceMatrix(duplicateDimension), /duplicate dimension/);

package/dist/test/production-runner.test.js

@@ -345,7 +345,7 @@ test("production runner uses its selected Node runtime for validation before obs
         assert.match(String(exit.cause), /missing-node/iu);
     }
 });
-test("production runner isolates qualification from a project SDK with a shorter hidden abort", async () => {
+test("production runner uses its bundled SDK with or without copy isolation", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-runner-isolated-sdk-"));
     roots.push(root);
     await writeShadowEvalSdk(root, 20);
@@ -383,16 +383,18 @@ test("production runner isolates qualification from a project SDK with a shorter
     const address = gateway.address();
     assert.ok(address !== null && typeof address !== "string");
     try {
-        const shadowed = await Effect.runPromise(withEvalService({
+        const unisolatedStarted = Date.now();
+        const unisolated = await Effect.runPromise(withEvalService({
             bearerCredential: "parent-only-token",
             timeoutMs: 15_000
         }, (service) => service.runComparison({
             ...requestFor(suite, `http://127.0.0.1:${String(address.port)}`),
             candidateModels: ["openai/cheap"],
             timeoutMs: 20
-        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici), Effect.exit));
-        assert.equal(Exit.isFailure(shadowed), true);
-        assert.deepEqual(calls, []);
+        }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
+        assert.deepEqual(calls, ["openai/cheap", "openai/judge"]);
+        assert.equal(unisolated.models[0]?.cases[0]?.outcome, "passed");
+        assert.ok(Date.now() - unisolatedStarted >= 75, "runner-owned SDK materialization must replace the project SDK's 20ms abort");
         const started = Date.now();
         const result = await Effect.runPromise(withEvalService({
             bearerCredential: "parent-only-token",
@@ -403,9 +405,14 @@ test("production runner isolates qualification from a project SDK with a shorter
             candidateModels: ["openai/cheap"],
             timeoutMs: 20
         }, "full")).pipe(Effect.provide(NodeHttpClient.layerUndici)));
-        assert.deepEqual(calls, ["openai/cheap", "openai/judge"]);
+        assert.deepEqual(calls, [
+            "openai/cheap",
+            "openai/judge",
+            "openai/cheap",
+            "openai/judge"
+        ]);
         assert.equal(result.models[0]?.cases[0]?.outcome, "passed");
-        assert.ok(Date.now() - started >= 75, "the project SDK's 20ms abort must not bound execution");
+        assert.ok(Date.now() - started >= 75, "copy isolation must also keep the project SDK's 20ms abort out of execution");
     }
     finally {
         await new Promise((resolve, reject) => gateway.close((error) => (error === undefined ? resolve() : reject(error))));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.0.24",
+  "version": "1.0.26",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.24",
-    "@velum-labs/routekit-eval-engine": "1.0.24",
-    "@velum-labs/routekit-eval-store": "1.0.24",
-    "@velum-labs/routekit-registry": "1.0.24"
+    "@velum-labs/routekit-eval-contracts": "1.0.26",
+    "@velum-labs/routekit-eval-engine": "1.0.26",
+    "@velum-labs/routekit-eval-store": "1.0.26",
+    "@velum-labs/routekit-registry": "1.0.26"
   },
   "devDependencies": {
     "typescript": "7.0.2"