@velum-labs/routekit-eval-service 1.0.18 → 1.0.20

package/dist/service.js

@@ -145,14 +145,38 @@ const enforceSpendLimit = (request, manifest) => {
     }
     return Effect.void;
 };
+const EXECUTION_MANIFEST_NAME = "routekit.eval-manifest.json";
+const discoverExecutionManifests = Effect.fnUntraced(function* (root) {
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    const directories = [root];
+    const manifests = [];
+    while (directories.length > 0) {
+        const directory = directories.pop();
+        const entries = yield* fs.readDirectory(directory);
+        for (const entry of entries) {
+            const entryPath = paths.join(directory, entry);
+            if (entry === EXECUTION_MANIFEST_NAME) {
+                manifests.push(entryPath);
+                continue;
+            }
+            const isSymbolicLink = yield* fs.readLink(entryPath).pipe(Effect.match({
+                onFailure: () => false,
+                onSuccess: () => true
+            }));
+            if (isSymbolicLink)
+                continue;
+            const info = yield* fs.stat(entryPath);
+            if (info.type === "Directory")
+                directories.push(entryPath);
+        }
+    }
+    return manifests;
+});
 const loadExecutionManifest = Effect.fn("EvalService.loadExecutionManifest")(function* (validation, request) {
     const fs = yield* FileSystem.FileSystem;
     const paths = yield* Path.Path;
-    const manifests = yield* fs
-        .glob("**/routekit.eval-manifest.json", {
-        root: validation.workingDirectory
-    })
-        .pipe(Effect.mapError((cause) => new EvalServiceValidationError({
+    const manifests = yield* discoverExecutionManifests(validation.workingDirectory).pipe(Effect.mapError((cause) => new EvalServiceValidationError({
         operation: "discover the comparison manifest",
         detail: detailOf(cause),
         cause

package/dist/test/service.test.js

@@ -1,12 +1,12 @@
 import assert from "node:assert/strict";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { EvalEngine } from "@velum-labs/routekit-eval-engine";
-import { Effect, Fiber, Layer } from "effect";
-import { EvalService, makeEvalServiceLayer } from "../service.js";
+import { Effect, Fiber, FileSystem, Layer, Path } from "effect";
+import { EvalService, makeEvalService, makeEvalServiceLayer } from "../service.js";
 const roots = [];
 after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
 const dimensions = [
@@ -97,6 +97,58 @@ const qualification = (root) => ({
     maximumUnknownWeight: 0.2,
     suites: suites(root)
 });
+test("manifest discovery does not call FileSystem.glob when Node glob is unavailable", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-manifest-no-glob-"));
+    roots.push(root);
+    const suiteDirectory = path.join(root, "nested", "suite");
+    const suitePath = path.join(suiteDirectory, "nested.eval.ts");
+    await mkdir(suiteDirectory, { recursive: true });
+    await writeFile(suitePath, 'import { test } from "node:test"; test("case", () => {});\n');
+    await symlink(root, path.join(suiteDirectory, "routekit-link"), "dir");
+    await writeFile(path.join(suiteDirectory, "routekit.eval-manifest.json"), `${JSON.stringify({
+        version: 1,
+        profileId: "nested",
+        candidateModels: ["openai/cheap", "anthropic/strong"],
+        judgeModel: "openai/judge",
+        caseCount: 1,
+        caseIds: ["case-1"],
+        maxOutputTokens: 256,
+        expectedCallCount: 4
+    })}\n`);
+    let globCalls = 0;
+    const inspection = await Effect.runPromise(Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const paths = yield* Path.Path;
+        const fileSystemWithoutNodeGlob = {
+            ...fs,
+            glob: () => {
+                globCalls += 1;
+                return Effect.die(new TypeError("Function.prototype.apply was called on undefined"));
+            }
+        };
+        const service = yield* makeEvalService().pipe(Effect.provideService(EvalEngine, EvalEngine.of({
+            discover: () => Effect.die("unexpected discover"),
+            validate: () => Effect.succeed({
+                searchRoot: suitePath,
+                workingDirectory: root,
+                files: [suitePath],
+                suiteDigest: "suite-nested"
+            }),
+            runComparison: () => Effect.die("unexpected comparison")
+        })), Effect.provideService(FileSystem.FileSystem, fileSystemWithoutNodeGlob), Effect.provideService(Path.Path, paths));
+        return yield* service.inspect({
+            version: 1,
+            profileId: "nested",
+            suitePath,
+            candidateModels: ["openai/cheap", "anthropic/strong"],
+            judgeModel: "openai/judge",
+            gatewayUrl: "http://127.0.0.1:8080/v1"
+        });
+    }).pipe(Effect.provide(NodeServicesLayer)));
+    assert.equal(inspection.suiteDigest, "suite-nested");
+    assert.equal(inspection.manifest.profileId, "nested");
+    assert.equal(globCalls, 0);
+});
 test("dimension matrix qualification inspects every manifest before publishing one activation", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-dimension-matrix-"));
     roots.push(root);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-service",
   "private": false,
-  "version": "1.0.18",
+  "version": "1.0.20",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,10 +32,10 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.108",
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.18",
-    "@velum-labs/routekit-eval-engine": "1.0.18",
-    "@velum-labs/routekit-eval-store": "1.0.18",
-    "@velum-labs/routekit-registry": "1.0.18"
+    "@velum-labs/routekit-eval-contracts": "1.0.20",
+    "@velum-labs/routekit-eval-engine": "1.0.20",
+    "@velum-labs/routekit-eval-store": "1.0.20",
+    "@velum-labs/routekit-registry": "1.0.20"
   },
   "devDependencies": {
     "typescript": "7.0.2"