@axlsdk/studio 0.10.0 → 0.10.1

package/dist/middleware.cjs

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/middleware.ts
@@ -24,9 +34,9 @@ __export(middleware_exports, {
   handleWsMessage: () => handleWsMessage
 });
 module.exports = __toCommonJS(middleware_exports);
-var import_node_path2 = require("path");
-var import_node_fs2 = require("fs");
-var import_node_url = require("url");
+var import_node_path3 = require("path");
+var import_node_fs3 = require("fs");
+var import_node_url2 = require("url");
 var import_node_server = require("@hono/node-server");
 var import_ws = require("ws");
 
@@ -308,8 +318,8 @@ var health_default = app;
 var import_hono2 = require("hono");
 var import_axl = require("@axlsdk/axl");
 function createWorkflowRoutes(connMgr) {
-  const app8 = new import_hono2.Hono();
-  app8.get("/workflows", (c) => {
+  const app7 = new import_hono2.Hono();
+  app7.get("/workflows", (c) => {
     const runtime = c.get("runtime");
     const workflows = runtime.getWorkflows().map((w) => ({
       name: w.name,
@@ -318,7 +328,7 @@ function createWorkflowRoutes(connMgr) {
     }));
     return c.json({ ok: true, data: workflows });
   });
-  app8.get("/workflows/:name", (c) => {
+  app7.get("/workflows/:name", (c) => {
     const runtime = c.get("runtime");
     const name = c.req.param("name");
     const workflow = runtime.getWorkflow(name);
@@ -337,7 +347,7 @@ function createWorkflowRoutes(connMgr) {
       }
     });
   });
-  app8.post("/workflows/:name/execute", async (c) => {
+  app7.post("/workflows/:name/execute", async (c) => {
     const runtime = c.get("runtime");
     const name = c.req.param("name");
     const workflow = runtime.getWorkflow(name);
@@ -368,7 +378,7 @@ function createWorkflowRoutes(connMgr) {
     const result = await runtime.execute(name, body.input ?? {}, { metadata: body.metadata });
     return c.json({ ok: true, data: { result } });
   });
-  return app8;
+  return app7;
 }
 
 // src/server/routes/executions.ts
@@ -402,8 +412,8 @@ var executions_default = app2;
 // src/server/routes/sessions.ts
 var import_hono4 = require("hono");
 function createSessionRoutes(connMgr) {
-  const app8 = new import_hono4.Hono();
-  app8.get("/sessions", async (c) => {
+  const app7 = new import_hono4.Hono();
+  app7.get("/sessions", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     if (!store.listSessions) {
@@ -417,7 +427,7 @@ function createSessionRoutes(connMgr) {
     }
     return c.json({ ok: true, data: sessions });
   });
-  app8.get("/sessions/:id", async (c) => {
+  app7.get("/sessions/:id", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     const id = c.req.param("id");
@@ -425,7 +435,7 @@ function createSessionRoutes(connMgr) {
     const handoffHistory = await store.getSessionMeta(id, "handoffHistory");
     return c.json({ ok: true, data: { id, history, handoffHistory: handoffHistory ?? [] } });
   });
-  app8.post("/sessions/:id/send", async (c) => {
+  app7.post("/sessions/:id/send", async (c) => {
     const runtime = c.get("runtime");
     const id = c.req.param("id");
     const body = await c.req.json();
@@ -433,7 +443,7 @@ function createSessionRoutes(connMgr) {
     const result = await session.send(body.workflow, body.message);
     return c.json({ ok: true, data: { result } });
   });
-  app8.post("/sessions/:id/stream", async (c) => {
+  app7.post("/sessions/:id/stream", async (c) => {
     const runtime = c.get("runtime");
     const id = c.req.param("id");
     const body = await c.req.json();
@@ -454,14 +464,14 @@ function createSessionRoutes(connMgr) {
     })();
     return c.json({ ok: true, data: { executionId, streaming: true } });
   });
-  app8.delete("/sessions/:id", async (c) => {
+  app7.delete("/sessions/:id", async (c) => {
     const runtime = c.get("runtime");
     const store = runtime.getStateStore();
     const id = c.req.param("id");
     await store.deleteSession(id);
     return c.json({ ok: true, data: { deleted: true } });
   });
-  return app8;
+  return app7;
 }
 
 // src/server/routes/agents.ts
@@ -694,61 +704,65 @@ var decisions_default = app6;
 // src/server/routes/costs.ts
 var import_hono9 = require("hono");
 function createCostRoutes(costAggregator) {
-  const app8 = new import_hono9.Hono();
-  app8.get("/costs", (c) => {
+  const app7 = new import_hono9.Hono();
+  app7.get("/costs", (c) => {
     return c.json({ ok: true, data: costAggregator.getData() });
   });
-  app8.post("/costs/reset", (c) => {
+  app7.post("/costs/reset", (c) => {
     costAggregator.reset();
     return c.json({ ok: true, data: { reset: true } });
   });
-  return app8;
+  return app7;
 }
 
 // src/server/routes/evals.ts
 var import_hono10 = require("hono");
-var app7 = new import_hono10.Hono();
-app7.get("/evals", async (c) => {
-  const runtime = c.get("runtime");
-  const evals = runtime.getRegisteredEvals();
-  return c.json({ ok: true, data: evals });
-});
-app7.post("/evals/:name/run", async (c) => {
-  const runtime = c.get("runtime");
-  const name = c.req.param("name");
-  const entry = runtime.getRegisteredEval(name);
-  if (!entry) {
-    return c.json(
-      { ok: false, error: { code: "NOT_FOUND", message: `Eval "${name}" not found` } },
-      404
-    );
-  }
-  try {
-    const result = await runtime.runRegisteredEval(name);
-    return c.json({ ok: true, data: result });
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
-  }
-});
-app7.post("/evals/compare", async (c) => {
-  const runtime = c.get("runtime");
-  const body = await c.req.json();
-  try {
-    const result = await runtime.evalCompare(body.baseline, body.candidate);
-    return c.json({ ok: true, data: result });
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
-  }
-});
-var evals_default = app7;
+function createEvalRoutes(evalLoader) {
+  const app7 = new import_hono10.Hono();
+  app7.get("/evals", async (c) => {
+    if (evalLoader) await evalLoader();
+    const runtime = c.get("runtime");
+    const evals = runtime.getRegisteredEvals();
+    return c.json({ ok: true, data: evals });
+  });
+  app7.post("/evals/:name/run", async (c) => {
+    if (evalLoader) await evalLoader();
+    const runtime = c.get("runtime");
+    const name = c.req.param("name");
+    const entry = runtime.getRegisteredEval(name);
+    if (!entry) {
+      return c.json(
+        { ok: false, error: { code: "NOT_FOUND", message: `Eval "${name}" not found` } },
+        404
+      );
+    }
+    try {
+      const result = await runtime.runRegisteredEval(name);
+      return c.json({ ok: true, data: result });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
+    }
+  });
+  app7.post("/evals/compare", async (c) => {
+    const runtime = c.get("runtime");
+    const body = await c.req.json();
+    try {
+      const result = await runtime.evalCompare(body.baseline, body.candidate);
+      return c.json({ ok: true, data: result });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return c.json({ ok: false, error: { code: "EVAL_ERROR", message } }, 400);
+    }
+  });
+  return app7;
+}
 
 // src/server/routes/playground.ts
 var import_hono11 = require("hono");
 function createPlaygroundRoutes(connMgr) {
-  const app8 = new import_hono11.Hono();
-  app8.post("/playground/chat", async (c) => {
+  const app7 = new import_hono11.Hono();
+  app7.post("/playground/chat", async (c) => {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     const workflowName = body.workflow ?? runtime.getWorkflowNames()[0];
@@ -779,20 +793,20 @@ function createPlaygroundRoutes(connMgr) {
       data: { sessionId, executionId, streaming: true }
     });
   });
-  return app8;
+  return app7;
 }
 
 // src/server/index.ts
 function createServer(options) {
   const { runtime, staticRoot, basePath = "", readOnly = false } = options;
-  const app8 = new import_hono12.Hono();
+  const app7 = new import_hono12.Hono();
   const connMgr = new ConnectionManager();
   const costAggregator = new CostAggregator(connMgr);
   if (options.cors !== false) {
-    app8.use("*", (0, import_cors.cors)());
+    app7.use("*", (0, import_cors.cors)());
   }
-  app8.use("*", errorHandler);
-  app8.use("*", async (c, next) => {
+  app7.use("*", errorHandler);
+  app7.use("*", async (c, next) => {
     c.set("runtime", runtime);
     await next();
   });
@@ -810,7 +824,7 @@ function createServer(options) {
       "POST /api/evals",
       "POST /api/playground"
     ];
-    app8.use("/api/*", async (c, next) => {
+    app7.use("/api/*", async (c, next) => {
       const apiIdx = c.req.path.indexOf("/api/");
       const apiPath = apiIdx >= 0 ? c.req.path.slice(apiIdx) : c.req.path;
       const key = `${c.req.method} ${apiPath}`;
@@ -836,9 +850,9 @@ function createServer(options) {
   api.route("/", memory_default);
   api.route("/", decisions_default);
   api.route("/", createCostRoutes(costAggregator));
-  api.route("/", evals_default);
+  api.route("/", createEvalRoutes(options.evalLoader));
   api.route("/", createPlaygroundRoutes(connMgr));
-  app8.route("/api", api);
+  app7.route("/api", api);
   const traceListener = (event) => {
     const traceEvent = event;
     if (traceEvent.executionId) {
@@ -851,7 +865,7 @@ function createServer(options) {
   };
   runtime.on("trace", traceListener);
   if (staticRoot) {
-    app8.use(
+    app7.use(
       "/*",
       (0, import_serve_static.serveStatic)({
         root: staticRoot,
@@ -876,14 +890,14 @@ function createServer(options) {
             "[axl-studio] Could not inject basePath into index.html \u2014 </head> tag not found. The SPA may not route correctly."
           );
         }
-        app8.get("*", (c) => c.html(injectedHtml));
+        app7.get("*", (c) => c.html(injectedHtml));
       }
     } else {
-      app8.get("*", (0, import_serve_static.serveStatic)({ root: staticRoot, path: "/index.html" }));
+      app7.get("*", (0, import_serve_static.serveStatic)({ root: staticRoot, path: "/index.html" }));
     }
   }
   return {
-    app: app8,
+    app: app7,
     connMgr,
     costAggregator,
     createWsHandlers: () => createWsHandlers(connMgr),
@@ -891,6 +905,175 @@ function createServer(options) {
   };
 }
 
+// src/eval-loader.ts
+var import_node_path2 = require("path");
+var import_node_fs2 = require("fs");
+var import_node_url = require("url");
+function createEvalLoader(config, runtime, cwd) {
+  let loadPromise;
+  const { patterns, conditions } = normalizeConfig(config);
+  const baseCwd = cwd ?? process.cwd();
+  return () => {
+    if (!loadPromise) {
+      loadPromise = loadEvalFiles(patterns, conditions, baseCwd, runtime).catch((err) => {
+        loadPromise = void 0;
+        throw err;
+      });
+    }
+    return loadPromise;
+  };
+}
+async function loadEvalFiles(patterns, conditions, cwd, runtime) {
+  if (conditions.length > 0) {
+    await registerConditions(conditions);
+  }
+  const files = resolvePatterns(patterns, cwd);
+  if (files.length === 0) {
+    console.warn(`[axl-studio] No eval files found matching: ${patterns.join(", ")}`);
+    return;
+  }
+  if (files.some((f) => /\.[mc]?tsx?$/.test(f))) {
+    await ensureTsxLoader();
+  }
+  for (const file of files) {
+    try {
+      const mod = await import((0, import_node_url.pathToFileURL)(file).href);
+      const evalConfig = mod.default ?? mod.config ?? mod;
+      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        console.warn(
+          `[axl-studio] Skipping ${file}: not a valid eval config (missing workflow, dataset, or scorers)`
+        );
+        continue;
+      }
+      const name = deriveEvalName(file, cwd);
+      if (runtime.getRegisteredEval(name)) {
+        console.warn(
+          `[axl-studio] Eval name "${name}" from ${file} collides with an already-registered eval \u2014 overwriting`
+        );
+      }
+      runtime.registerEval(name, evalConfig, mod.executeWorkflow);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[axl-studio] Failed to load eval ${file}: ${msg}`);
+    }
+  }
+}
+function normalizeConfig(config) {
+  if (typeof config === "string") {
+    return { patterns: [config], conditions: [] };
+  }
+  if (Array.isArray(config)) {
+    return { patterns: config, conditions: [] };
+  }
+  const files = typeof config.files === "string" ? [config.files] : config.files;
+  return { patterns: files, conditions: config.conditions ?? [] };
+}
+function deriveEvalName(filePath, cwd) {
+  const rel = (0, import_node_path2.relative)(cwd, filePath);
+  const normalized = rel.replace(/\\/g, "/");
+  if (normalized.startsWith("../")) {
+    const base = (0, import_node_path2.basename)(filePath);
+    const stripped = base.replace(/\.eval\.[mc]?[jt]sx?$/, "");
+    return stripped !== base ? stripped : base.replace(/\.[mc]?[jt]sx?$/, "") || base;
+  }
+  const withoutEval = normalized.replace(/\.eval\.[mc]?[jt]sx?$/, "");
+  if (withoutEval !== normalized) return withoutEval;
+  const withoutExt = normalized.replace(/\.[mc]?[jt]sx?$/, "");
+  return withoutExt || normalized;
+}
+function resolvePatterns(patterns, cwd) {
+  const files = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const pattern of patterns) {
+    const resolved = pattern.includes("*") ? expandGlob(pattern, cwd) : [(0, import_node_path2.resolve)(cwd, pattern)];
+    for (const file of resolved) {
+      if (!seen.has(file)) {
+        seen.add(file);
+        files.push(file);
+      }
+    }
+  }
+  return files;
+}
+function expandGlob(pattern, cwd) {
+  if (pattern.includes("**/")) {
+    const sepIdx = pattern.indexOf("**/");
+    const baseDir = (0, import_node_path2.resolve)(cwd, pattern.slice(0, sepIdx) || ".");
+    const fileGlob2 = pattern.slice(sepIdx + 3) || "*";
+    return findFiles(baseDir, fileGlob2, true);
+  }
+  const dir = (0, import_node_path2.resolve)(cwd, (0, import_node_path2.dirname)(pattern));
+  const fileGlob = (0, import_node_path2.basename)(pattern);
+  return findFiles(dir, fileGlob, false);
+}
+var MAX_DEPTH = 20;
+function findFiles(dir, fileGlob, recursive, depth = 0) {
+  if (depth > MAX_DEPTH) return [];
+  const matcher = globToRegex(fileGlob);
+  const results = [];
+  try {
+    const entries = (0, import_node_fs2.readdirSync)(dir);
+    for (const entry of entries) {
+      const full = (0, import_node_path2.resolve)(dir, entry);
+      try {
+        const stat = (0, import_node_fs2.statSync)(full);
+        if (stat.isFile() && matcher.test(entry)) {
+          results.push(full);
+        } else if (stat.isDirectory() && recursive) {
+          results.push(...findFiles(full, fileGlob, true, depth + 1));
+        }
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return results;
+}
+function globToRegex(glob) {
+  const escaped = glob.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${escaped}$`);
+}
+var tsxRegistered = false;
+async function ensureTsxLoader() {
+  if (tsxRegistered) return;
+  tsxRegistered = true;
+  let loaded = false;
+  try {
+    const tsxEsm = await import("tsx/esm/api");
+    tsxEsm.register();
+    loaded = true;
+  } catch {
+  }
+  try {
+    const tsxCjs = await import("tsx/cjs/api");
+    tsxCjs.register();
+    loaded = true;
+  } catch {
+  }
+  if (!loaded) {
+    console.warn(
+      "[axl-studio] Warning: tsx is not installed. TypeScript eval files require tsx.\n  Install it with: npm install -D tsx"
+    );
+  }
+}
+async function registerConditions(conditions) {
+  try {
+    const nodeModule = await import("module");
+    const hookCode = [
+      `const extra = ${JSON.stringify(conditions)};`,
+      `export async function resolve(specifier, context, nextResolve) {`,
+      `  return nextResolve(specifier, {`,
+      `    ...context,`,
+      `    conditions: [...new Set([...context.conditions, ...extra])],`,
+      `  });`,
+      `}`
+    ].join("\n");
+    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
+  } catch {
+    console.warn("[axl-studio] Warning: import conditions require Node.js 20.6+");
+  }
+}
+
 // src/middleware.ts
 var import_meta = {};
 function createStudioMiddleware(options) {
@@ -898,25 +1081,27 @@ function createStudioMiddleware(options) {
   const basePath = normalizeBasePath(options.basePath);
   const staticRoot = serveClient ? resolveClientDist() : void 0;
   if (serveClient && !staticRoot) {
-    const dir = import_meta.dirname ?? (0, import_node_path2.dirname)((0, import_node_url.fileURLToPath)(import_meta.url));
+    const dir = import_meta.dirname ?? (0, import_node_path3.dirname)((0, import_node_url2.fileURLToPath)(import_meta.url));
     console.warn(
-      `[axl-studio] serveClient is true but no pre-built client found at ${(0, import_node_path2.resolve)(dir, "client")}. Studio UI will not be available. Set serveClient: false to suppress this warning.`
+      `[axl-studio] serveClient is true but no pre-built client found at ${(0, import_node_path3.resolve)(dir, "client")}. Studio UI will not be available. Set serveClient: false to suppress this warning.`
     );
   }
-  const { app: app8, connMgr, traceListener } = createServer({
+  const evalLoader = options.evals ? createEvalLoader(options.evals, runtime) : void 0;
+  const { app: app7, connMgr, traceListener } = createServer({
     runtime,
     staticRoot,
     basePath,
     readOnly,
-    cors: false
+    cors: false,
     // Host framework owns CORS policy
+    evalLoader
   });
   if (process.env.NODE_ENV === "production" && !verifyUpgrade) {
     console.warn(
       "[axl-studio] WARNING: Studio middleware mounted in production without verifyUpgrade. WebSocket connections are not authenticated. All registered workflows, tools, and agents are accessible. See https://axlsdk.com/docs/studio/security"
     );
   }
-  const listener = (0, import_node_server.getRequestListener)(app8.fetch, {
+  const listener = (0, import_node_server.getRequestListener)(app7.fetch, {
     overrideGlobalObjects: false
   });
   let closed = false;
@@ -1020,7 +1205,7 @@ function createStudioMiddleware(options) {
     handler,
     handleWebSocket,
     upgradeWebSocket,
-    app: app8,
+    app: app7,
     connectionManager: connMgr,
     close
   };
@@ -1046,9 +1231,9 @@ function normalizeBasePath(raw) {
   return normalized;
 }
 function resolveClientDist() {
-  const dir = import_meta.dirname ?? (0, import_node_path2.dirname)((0, import_node_url.fileURLToPath)(import_meta.url));
-  const candidate = (0, import_node_path2.resolve)(dir, "client");
-  return (0, import_node_fs2.existsSync)((0, import_node_path2.resolve)(candidate, "index.html")) ? candidate : void 0;
+  const dir = import_meta.dirname ?? (0, import_node_path3.dirname)((0, import_node_url2.fileURLToPath)(import_meta.url));
+  const candidate = (0, import_node_path3.resolve)(dir, "client");
+  return (0, import_node_fs3.existsSync)((0, import_node_path3.resolve)(candidate, "index.html")) ? candidate : void 0;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {