@axlsdk/studio 0.16.1 → 0.17.0

package/dist/middleware.js

@@ -1,22 +1,19 @@
-import {
-  importModule
-} from "./chunk-JGQ3MSIG.js";
 import {
   createServer,
   handleWsMessage
-} from "./chunk-RE6VPUXA.js";
+} from "./chunk-GADFO7DZ.js";
 
 // src/middleware.ts
-import { resolve as resolve2, dirname as dirname2 } from "path";
+import { resolve as resolve2, dirname } from "path";
 import { existsSync } from "fs";
 import { fileURLToPath } from "url";
 import { getRequestListener } from "@hono/node-server";
 import { WebSocketServer } from "ws";
 
 // src/eval-loader.ts
-import { resolve, relative, dirname, basename } from "path";
-import { readdirSync, statSync } from "fs";
+import { resolve, relative, basename } from "path";
 import { pathToFileURL } from "url";
+import { importModule, expandGlob, registerConditions, pickDefault, pickExport } from "@axlsdk/axl";
 var parentURL = import.meta.url ?? pathToFileURL(typeof __filename !== "undefined" ? __filename : __dirname).href;
 function createEvalLoader(config, runtime, cwd) {
   let loadPromise;
@@ -44,10 +41,29 @@ async function loadEvalFiles(patterns, conditions, cwd, runtime) {
   for (const file of files) {
     try {
       const mod = await importModule(file, parentURL);
-      const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
+      const evalConfig = pickDefault(mod);
+      if (!evalConfig || typeof evalConfig !== "object") {
+        console.warn(
+          `[axl-studio] Skipping ${file}: default export is ${evalConfig === null ? "null" : typeof evalConfig}, expected an eval config object.`
+        );
+        continue;
+      }
+      if (Array.isArray(evalConfig.scorers) && evalConfig.scorers.length === 0) {
+        console.warn(
+          `[axl-studio] Skipping ${file}: scorers array is empty \u2014 at least one scorer is required.`
+        );
+        continue;
+      }
       if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        const missing = [
+          !evalConfig.workflow && "workflow",
+          !evalConfig.dataset && "dataset",
+          !evalConfig.scorers && "scorers"
+        ].filter(Boolean).join(", ");
+        const keys = Object.keys(evalConfig).slice(0, 10);
+        const got = keys.length > 0 ? ` Got: { ${keys.join(", ")} }.` : "";
         console.warn(
-          `[axl-studio] Skipping ${file}: not a valid eval config (missing workflow, dataset, or scorers)`
+          `[axl-studio] Skipping ${file}: not a valid eval config (missing ${missing}).${got}`
         );
         continue;
       }
@@ -57,7 +73,16 @@ async function loadEvalFiles(patterns, conditions, cwd, runtime) {
           `[axl-studio] Eval name "${name}" from ${file} collides with an already-registered eval \u2014 overwriting`
         );
       }
-      runtime.registerEval(name, evalConfig, mod.executeWorkflow);
+      const exported = pickExport(mod, "executeWorkflow");
+      let customExecute;
+      if (typeof exported === "function") {
+        customExecute = exported;
+      } else if (exported !== void 0) {
+        console.warn(
+          `[axl-studio] ${file} exports executeWorkflow but it is ${typeof exported}, not a function \u2014 ignoring and falling back to runtime.execute()`
+        );
+      }
+      runtime.registerEval(name, evalConfig, customExecute);
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       console.warn(`[axl-studio] Failed to load eval ${file}: ${msg}`);
@@ -101,61 +126,6 @@ function resolvePatterns(patterns, cwd) {
   }
   return files;
 }
-function expandGlob(pattern, cwd) {
-  if (pattern.includes("**/")) {
-    const sepIdx = pattern.indexOf("**/");
-    const baseDir = resolve(cwd, pattern.slice(0, sepIdx) || ".");
-    const fileGlob2 = pattern.slice(sepIdx + 3) || "*";
-    return findFiles(baseDir, fileGlob2, true);
-  }
-  const dir = resolve(cwd, dirname(pattern));
-  const fileGlob = basename(pattern);
-  return findFiles(dir, fileGlob, false);
-}
-var MAX_DEPTH = 20;
-function findFiles(dir, fileGlob, recursive, depth = 0) {
-  if (depth > MAX_DEPTH) return [];
-  const matcher = globToRegex(fileGlob);
-  const results = [];
-  try {
-    const entries = readdirSync(dir);
-    for (const entry of entries) {
-      const full = resolve(dir, entry);
-      try {
-        const stat = statSync(full);
-        if (stat.isFile() && matcher.test(entry)) {
-          results.push(full);
-        } else if (stat.isDirectory() && recursive) {
-          results.push(...findFiles(full, fileGlob, true, depth + 1));
-        }
-      } catch {
-      }
-    }
-  } catch {
-  }
-  return results;
-}
-function globToRegex(glob) {
-  const escaped = glob.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  return new RegExp(`^${escaped}$`);
-}
-async function registerConditions(conditions) {
-  try {
-    const nodeModule = await import("module");
-    const hookCode = [
-      `const extra = ${JSON.stringify(conditions)};`,
-      `export async function resolve(specifier, context, nextResolve) {`,
-      `  return nextResolve(specifier, {`,
-      `    ...context,`,
-      `    conditions: [...new Set([...context.conditions, ...extra])],`,
-      `  });`,
-      `}`
-    ].join("\n");
-    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
-  } catch {
-    console.warn("[axl-studio] Warning: import conditions require Node.js 20.6+");
-  }
-}
 
 // src/middleware.ts
 function createStudioMiddleware(options) {
@@ -169,7 +139,7 @@ function createStudioMiddleware(options) {
   const basePath = normalizeBasePath(options.basePath);
   const staticRoot = serveClient ? resolveClientDist() : void 0;
   if (serveClient && !staticRoot) {
-    const dir = import.meta.dirname ?? (typeof __dirname !== "undefined" ? __dirname : dirname2(fileURLToPath(import.meta.url)));
+    const dir = import.meta.dirname ?? (typeof __dirname !== "undefined" ? __dirname : dirname(fileURLToPath(import.meta.url)));
     console.warn(
       `[axl-studio] serveClient is true but no pre-built client found at ${resolve2(dir, "client")}. Studio UI will not be available. Set serveClient: false to suppress this warning.`
     );
@@ -344,7 +314,7 @@ function normalizeBasePath(raw) {
   return normalized;
 }
 function resolveClientDist() {
-  const dir = import.meta.dirname ?? (typeof __dirname !== "undefined" ? __dirname : dirname2(fileURLToPath(import.meta.url)));
+  const dir = import.meta.dirname ?? (typeof __dirname !== "undefined" ? __dirname : dirname(fileURLToPath(import.meta.url)));
   const candidate = resolve2(dir, "client");
   return existsSync(resolve2(candidate, "index.html")) ? candidate : void 0;
 }

package/dist/middleware.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/middleware.ts","../src/eval-loader.ts"],"sourcesContent":["import { resolve, dirname } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { fileURLToPath } from 'node:url';\nimport { getRequestListener } from '@hono/node-server';\nimport { WebSocketServer } from 'ws';\nimport { createServer } from './server/index.js';\nimport { handleWsMessage } from './server/ws/protocol.js';\nimport { createEvalLoader } from './eval-loader.js';\nimport type { EvalLoaderConfig } from './eval-loader.js';\nimport type { BufferCaps } from './server/ws/connection-manager.js';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport type { IncomingMessage, ServerResponse, Server } from 'node:http';\n\nexport type { EvalLoaderConfig } from './eval-loader.js';\nexport type { BufferCaps } from './server/ws/connection-manager.js';\n\nexport type StudioMiddlewareOptions = {\n  /** The AxlRuntime instance to observe and control. */\n  runtime: AxlRuntime;\n\n  /**\n   * URL path prefix where Studio is mounted.\n   * Must match the mount path in your framework (Express `app.use()`,\n   * Fastify `register()`, Hono `app.route()`, etc.). The framework is\n   * expected to strip the prefix from `req.url` before calling the handler.\n   *\n   * Do not set basePath when using a raw `http.Server` as the root handler —\n   * leave it empty and mount at root instead.\n   *\n   * Must start with '/' when non-empty. Trailing slashes are stripped.\n   * Only URL-safe characters allowed: [a-zA-Z0-9/_-]\n   *\n   * @example '/studio'\n   * @example '/admin/studio'\n   * @example ''  — mounted at root (default)\n   */\n  basePath?: string;\n\n  /**\n   * Serve the pre-built Studio SPA.\n   * Set to false if serving the client from a CDN or separate build.\n   * @default true\n   */\n  serveClient?: boolean;\n\n  /**\n   * Verify a WebSocket upgrade request before completing the handshake.\n   *\n   * Return either a boolean (`true` allow, `false` reject) or an object\n   * `{ allowed: boolean, metadata?: unknown }` to ALSO attach per-connection\n   * metadata that `filterTraceEvent` can later read — typically the auth\n   * subject (userId, tenantId, role). Throw to reject with an error.\n   *\n   * IMPORTANT: WebSocket upgrades bypass Express/Fastify/Koa middleware.\n   * If your HTTP routes are behind auth middleware, WS connections are NOT\n   * automatically protected. Use this callback to enforce authentication\n   * on WebSocket connections.\n   *\n   * @example Multi-tenant: extract tenant from JWT, attach as metadata\n   * verifyUpgrade: async (req) => {\n   *   const token = req.headers.authorization?.slice(7);\n   *   const claims = await verifyJwt(token);\n   *   if (!claims) return false;\n   *   return { allowed: true, metadata: { userId: claims.sub, tenantId: claims.tid } };\n   * }\n   */\n  verifyUpgrade?: (\n    req: IncomingMessage,\n  ) =>\n    | boolean\n    | { allowed: boolean; metadata?: unknown }\n    | Promise<boolean | { allowed: boolean; metadata?: unknown }>;\n\n  /**\n   * Optional per-event filter used by multi-tenant deployments to scope the\n   * trace firehose. Called on every outbound WebSocket broadcast; return\n   * `true` to deliver to this connection, `false` to drop.\n   *\n   * `event` is the parsed event payload — for the `trace:*` channel this is\n   * an `AxlEvent` from `@axlsdk/axl` (narrow via the discriminated union);\n   * for `costs` it's a `CostData`; for `execution:*` / `eval:*` it's the\n   * legacy stream-event shape (until PR 3 collapses the wire to `AxlEvent`).\n   * Typed `unknown` because the filter runs for every channel.\n   * `metadata` is the per-connection metadata attached by `verifyUpgrade`,\n   * or `undefined` if `verifyUpgrade` returned a bare boolean.\n   *\n   * Filter errors are treated as `drop` (fail-closed) so a buggy predicate\n   * cannot accidentally leak events cross-tenant.\n   *\n   * @example Scope trace events by tenant id stored on agent metadata\n   * import type { AxlEvent } from '@axlsdk/axl';\n   *\n   * filterTraceEvent: (event, meta) => {\n   *   const m = meta as { tenantId?: string } | undefined;\n   *   if (!m?.tenantId) return false;\n   *   const e = event as AxlEvent;\n   *   // Only narrow event.type === 'agent_call_end' events; pass structural\n   *   // events (workflow_start/end, cost updates) through unconditionally.\n   *   if (e.type !== 'agent_call_end') return true;\n   *   return e.workflow?.startsWith(`${m.tenantId}:`) ?? false;\n   * }\n   */\n  filterTraceEvent?: (event: unknown, metadata: unknown) => boolean;\n\n  /**\n   * Disable all mutating endpoints (execute, test, send, delete, resolve).\n   * When true, Studio is observation-only.\n   * @default false\n   */\n  readOnly?: boolean;\n\n  /**\n   * Lazy-load eval files for the Eval Runner panel.\n   *\n   * Eval files are dynamically imported on first access to eval endpoints,\n   * not at middleware construction time. This means:\n   * - Zero cost during normal API operation\n   * - Eval files can import from any module without creating circular deps\n   *   in the static module graph (they're loaded as standalone entry points)\n   * - `@axlsdk/eval` can remain a devDependency — eval files never enter\n   *   the production bundle since bundlers can't see dynamic imports\n   *\n   * Accepts glob patterns, explicit file paths, or an object with\n   * `conditions` for monorepo source export resolution.\n   *\n   * Eval files are loaded once and cached for the middleware's lifetime.\n   * Changes to eval files require a server restart.\n   *\n   * @example\n   * // Single glob pattern\n   * evals: 'evals/*.eval.ts'\n   *\n   * @example\n   * // Multiple patterns\n   * evals: ['libs/api/evals/*.eval.ts', 'libs/ai/evals/*.eval.ts']\n   *\n   * @example\n   * // With import conditions for monorepo source exports\n   * evals: {\n   *   files: 'libs/api/evals/*.eval.ts',\n   *   conditions: ['development'],\n   * }\n   */\n  evals?: EvalLoaderConfig;\n\n  /**\n   * Override the default WebSocket replay-buffer resource caps for\n   * production deployments under sustained execution churn.\n   *\n   * Defaults: `maxEventsPerBuffer: 1000`, `maxBytesPerBuffer: 4 MiB`,\n   * `maxActiveBuffers: 256`. Worst-case memory is roughly\n   * `maxActiveBuffers × maxBytesPerBuffer` (≈1 GiB at defaults). Tighten\n   * either dimension to lower the ceiling at the cost of late-subscriber\n   * replay coverage; raise them if you need more event headroom on long\n   * verbose-mode streams.\n   *\n   * Terminal `done` / `error` events are always buffered regardless of\n   * caps, so a late subscriber to a completed stream still sees the\n   * outcome — only intermediate non-terminal events are dropped.\n   *\n   * All fields are optional; passing `{}` is a no-op.\n   *\n   * @example\n   * // Halve memory ceiling on a high-churn deployment\n   * bufferCaps: { maxActiveBuffers: 128, maxBytesPerBuffer: 2 * 1024 * 1024 }\n   */\n  bufferCaps?: BufferCaps;\n};\n\n/**\n * Minimal contract a WebSocket connection must satisfy.\n * Matches the `ws` library API (de facto standard in Node.js).\n */\nexport interface StudioWebSocket {\n  send(data: string): void;\n  close(): void;\n  on(event: 'message', fn: (data: string | Buffer) => void): void;\n  on(event: 'close', fn: () => void): void;\n  on(event: 'error', fn: (err: Error) => void): void;\n}\n\n// Re-export for Hono-in-Hono consumers\nexport { handleWsMessage } from './server/ws/protocol.js';\n\nexport type StudioMiddleware = ReturnType<typeof createStudioMiddleware>;\n\nexport function createStudioMiddleware(options: StudioMiddlewareOptions) {\n  const {\n    runtime,\n    serveClient = true,\n    verifyUpgrade,\n    readOnly = false,\n    filterTraceEvent,\n  } = options;\n\n  // Normalize basePath: strip trailing slashes, validate format\n  const basePath = normalizeBasePath(options.basePath);\n\n  // Resolve pre-built SPA assets from this package's dist/\n  const staticRoot = serveClient ? resolveClientDist() : undefined;\n\n  if (serveClient && !staticRoot) {\n    const dir =\n      import.meta.dirname ??\n      (typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url)));\n    console.warn(\n      '[axl-studio] serveClient is true but no pre-built client found at ' +\n        `${resolve(dir, 'client')}. Studio UI will not be available. ` +\n        'Set serveClient: false to suppress this warning.',\n    );\n  }\n\n  // Create lazy eval loader if eval files are configured\n  const evalLoader = options.evals ? createEvalLoader(options.evals, runtime) : undefined;\n\n  const { app, connMgr, traceListener, closeActiveRuns, closeAggregators } = createServer({\n    runtime,\n    staticRoot,\n    basePath,\n    readOnly,\n    cors: false, // Host framework owns CORS policy\n    evalLoader,\n    bufferCaps: options.bufferCaps,\n  });\n\n  // Install the broadcast filter once at construction time. The connMgr\n  // applies it on every outbound event; `metadata` is attached to each\n  // connection after a successful `verifyUpgrade`.\n  if (filterTraceEvent) {\n    connMgr.setFilter(filterTraceEvent);\n  }\n\n  // Log production safety warning\n  if (process.env.NODE_ENV === 'production' && !verifyUpgrade) {\n    console.warn(\n      '[axl-studio] WARNING: Studio middleware mounted in production without verifyUpgrade. ' +\n        'WebSocket connections are not authenticated. All registered workflows, tools, and ' +\n        'agents are accessible. See https://axlsdk.com/docs/studio/security',\n    );\n  }\n\n  // Convert Hono app → Node.js (req, res) handler via @hono/node-server.\n  // overrideGlobalObjects: false prevents replacing global.Request/Response,\n  // which could break the host application.\n  const listener = getRequestListener(app.fetch, {\n    overrideGlobalObjects: false,\n  });\n\n  let closed = false;\n\n  function handler(req: IncomingMessage, res: ServerResponse) {\n    if (closed) {\n      res.writeHead(503, { 'Content-Type': 'application/json' });\n      res.end(\n        JSON.stringify({\n          ok: false,\n          error: { code: 'CLOSED', message: 'Studio middleware has been shut down' },\n        }),\n      );\n      return;\n    }\n\n    // Express/NestJS/Koa body parsers consume the raw IncomingMessage stream\n    // and store the parsed result on req.body. When that happens,\n    // @hono/node-server's getRequestListener reads an empty stream and Hono\n    // sees no body. To fix this, we re-serialize req.body as req.rawBody —\n    // a Buffer that getRequestListener checks before falling back to the\n    // stream (see newRequestFromIncoming in @hono/node-server/dist/listener.js).\n    //\n    // Verified against @hono/node-server@1.19.9. If this breaks after an\n    // upgrade, check whether the rawBody instanceof Buffer check still exists\n    // in newRequestFromIncoming.\n    const reqAny = req as unknown as Record<string, unknown>;\n    if (reqAny.body != null && !reqAny.rawBody) {\n      try {\n        reqAny.rawBody = Buffer.from(JSON.stringify(reqAny.body));\n      } catch {\n        // Non-serializable body — Hono will see an empty body\n      }\n    }\n\n    listener(req, res).catch((err) => {\n      console.error('[axl-studio] Unhandled error in request handler:', err);\n      if (!res.headersSent) {\n        res.writeHead(500, { 'Content-Type': 'application/json' });\n        res.end(\n          JSON.stringify({\n            ok: false,\n            error: { code: 'INTERNAL_ERROR', message: 'Internal server error' },\n          }),\n        );\n      }\n    });\n  }\n\n  // Handle an individual WebSocket using the Studio protocol.\n  // Adapts any StudioWebSocket to ConnectionManager's internal BroadcastTarget.\n  // `metadata` is whatever `verifyUpgrade` attached — passed through so the\n  // `filterTraceEvent` callback can read it on every outbound event.\n  function handleWebSocket(ws: StudioWebSocket, metadata?: unknown) {\n    if (closed) {\n      ws.close();\n      return;\n    }\n    const socket = {\n      send: (data: string) => ws.send(data),\n      close: () => ws.close(),\n    };\n    connMgr.add(socket);\n    if (metadata !== undefined) {\n      connMgr.setMetadata(socket, metadata);\n    }\n\n    ws.on('message', (raw) => {\n      const reply = handleWsMessage(String(raw), socket, connMgr);\n      if (reply) ws.send(reply);\n    });\n\n    ws.on('close', () => connMgr.remove(socket));\n    ws.on('error', () => connMgr.remove(socket));\n  }\n\n  // Internal WebSocketServer — created lazily by upgradeWebSocket()\n  let wss: InstanceType<typeof WebSocketServer> | undefined;\n  // References for cleanup: the upgrade handler and server it's attached to\n  let upgradeHandler: ((...args: any[]) => void) | undefined;\n  let serverRef: Server | undefined;\n\n  // Convenience: attach WS handling to an http.Server.\n  function upgradeWebSocket(server: Server, path?: string) {\n    if (wss) {\n      throw new Error(\n        '[axl-studio] upgradeWebSocket() has already been called. ' +\n          'Call close() first if you need to re-attach.',\n      );\n    }\n\n    const wsPath = path ?? (basePath ? `${basePath}/ws` : '/ws');\n\n    wss = new WebSocketServer({ noServer: true });\n    serverRef = server;\n\n    upgradeHandler = async (req: IncomingMessage, socket: any, head: Buffer) => {\n      // Match path, ignoring query string\n      const pathname = new URL(req.url!, `http://${req.headers.host}`).pathname;\n      if (pathname !== wsPath) return; // Let other upgrade handlers run\n\n      // Early-reject if close() already started. This is a cheap pre-check —\n      // we repeat it AFTER the async verifyUpgrade resolves to close the race\n      // where close() runs while verifyUpgrade is still in flight.\n      if (closed) {\n        socket.destroy();\n        return;\n      }\n\n      // Apply auth verification and capture optional per-connection metadata\n      // (e.g. { userId, tenantId }) that filterTraceEvent can later read.\n      let connectionMetadata: unknown;\n      if (verifyUpgrade) {\n        try {\n          const result = await verifyUpgrade(req);\n          // Normalize both return shapes: plain boolean or { allowed, metadata }\n          const allowed = typeof result === 'boolean' ? result : result.allowed;\n          if (!allowed) {\n            socket.write('HTTP/1.1 401 Unauthorized\\r\\n\\r\\n');\n            socket.destroy();\n            return;\n          }\n          if (typeof result === 'object' && result !== null) {\n            connectionMetadata = result.metadata;\n          }\n        } catch {\n          socket.write('HTTP/1.1 403 Forbidden\\r\\n\\r\\n');\n          socket.destroy();\n          return;\n        }\n      }\n\n      // Guard against close() having run during async verifyUpgrade. We check\n      // BOTH `closed` (set at the top of `close()`) and `!wss` (set at the\n      // end). `closed` is the authoritative signal — checking `!wss` alone\n      // wouldn't catch the window where `close()` has marked the middleware\n      // as shut down but hasn't yet torn down the WebSocketServer. Failing\n      // open here would leak a connection past the middleware's lifetime\n      // and leave it holding references the host expected to be released.\n      if (closed || !wss) {\n        socket.destroy();\n        return;\n      }\n\n      wss.handleUpgrade(req, socket, head, (ws) => {\n        handleWebSocket(ws, connectionMetadata);\n      });\n    };\n\n    server.on('upgrade', upgradeHandler);\n  }\n\n  // Cleanup function for lifecycle management.\n  function close() {\n    closed = true;\n\n    // Abort active streaming eval runs before closing connections\n    closeActiveRuns();\n\n    // Close all WebSocket connections\n    connMgr.closeAll();\n\n    // Remove the upgrade listener from the server before closing WSS\n    if (upgradeHandler && serverRef) {\n      serverRef.removeListener('upgrade', upgradeHandler);\n      upgradeHandler = undefined;\n      serverRef = undefined;\n    }\n\n    // Shut down the internal WebSocketServer if one was created\n    if (wss) {\n      wss.close();\n      wss = undefined;\n    }\n\n    // Remove only our trace event listener from the runtime\n    if (traceListener) {\n      runtime.removeListener('trace', traceListener);\n    }\n\n    // Close all aggregators (clear intervals and unsubscribe listeners)\n    closeAggregators();\n  }\n\n  return {\n    handler,\n    handleWebSocket,\n    upgradeWebSocket,\n    app,\n    connectionManager: connMgr,\n    close,\n  };\n}\n\n/**\n * Normalize and validate basePath.\n * - Empty string and undefined → ''\n * - Strip trailing slashes\n * - Validate leading slash when non-empty\n * - Reject unsafe characters\n */\nfunction normalizeBasePath(raw?: string): string {\n  if (!raw) return '';\n\n  // Strip trailing slashes\n  const normalized = raw.replace(/\\/+$/, '');\n  if (!normalized) return '';\n\n  // Must start with /\n  if (!normalized.startsWith('/')) {\n    throw new Error(`basePath must start with '/' (got '${raw}'). Example: '/studio'`);\n  }\n\n  // Reject path traversal, consecutive slashes, and unsafe characters\n  if (normalized.includes('..')) {\n    throw new Error(`basePath must not contain '..' segments (got '${raw}')`);\n  }\n  if (normalized.includes('//')) {\n    throw new Error(`basePath must not contain consecutive slashes (got '${raw}')`);\n  }\n  if (!/^\\/[a-zA-Z0-9/_-]*$/.test(normalized)) {\n    throw new Error(\n      `basePath contains invalid characters (got '${raw}'). ` +\n        'Only alphanumeric characters, /, _, and - are allowed.',\n    );\n  }\n\n  return normalized;\n}\n\nfunction resolveClientDist(): string | undefined {\n  // Resolve the directory of this file (dist/ in published package).\n  const dir =\n    import.meta.dirname ??\n    (typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url)));\n  const candidate = resolve(dir, 'client');\n  return existsSync(resolve(candidate, 'index.html')) ? candidate : undefined;\n}\n","import { resolve, relative, dirname, basename } from 'node:path';\nimport { readdirSync, statSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { importModule } from './cli-utils.js';\n\n// In the CJS bundle, tsup stubs import.meta as {} so import.meta.url is\n// undefined. Fall back to __filename (which CJS defines) converted to a\n// file:// URL so tsImport() gets a valid parentURL.\nconst parentURL: string =\n  import.meta.url ?? pathToFileURL(typeof __filename !== 'undefined' ? __filename : __dirname).href;\n\n/**\n * Configuration for lazy eval file discovery.\n *\n * - `string` — a glob pattern or explicit file path\n * - `string[]` — multiple patterns/paths\n * - `object` — patterns with optional import conditions\n */\nexport type EvalLoaderConfig =\n  | string\n  | string[]\n  | {\n      files: string | string[];\n\n      /**\n       * Custom Node.js import conditions (e.g., `['development']`).\n       *\n       * In monorepos, package.json `exports` often use the `development` condition\n       * to point at source (`.ts`) instead of built dist. Without this, eval files\n       * that import workspace packages resolve to dist files, which may not exist.\n       *\n       * **WARNING**: Conditions are registered process-wide via `module.register()`.\n       * They affect all subsequent imports in the process, not just eval files.\n       */\n      conditions?: string[];\n    };\n\n/**\n * Create a lazy eval loader that resolves file patterns and dynamically imports\n * eval files on first call, registering them with the runtime.\n *\n * The loader is idempotent — subsequent calls return the same promise.\n * Concurrent callers all await the same loading work.\n *\n * Eval files should export a default config with `{ workflow, dataset, scorers }`\n * (the result of `defineEval()` from `@axlsdk/eval`). An optional named export\n * `executeWorkflow` overrides the default `runtime.execute()` behavior.\n *\n * Eval names are the file's path relative to `cwd` (project root), minus the\n * `.eval.*` suffix. This makes names completely stable — a file's name never\n * changes regardless of what other files or patterns exist.\n *\n * @param config  Glob patterns, file paths, or object with conditions\n * @param runtime The AxlRuntime to register discovered evals on\n * @param cwd     Base directory for resolving patterns and deriving names (default: `process.cwd()`)\n */\nexport function createEvalLoader(\n  config: EvalLoaderConfig,\n  runtime: AxlRuntime,\n  cwd?: string,\n): () => Promise<void> {\n  let loadPromise: Promise<void> | undefined;\n  const { patterns, conditions } = normalizeConfig(config);\n  const baseCwd = cwd ?? process.cwd();\n\n  return () => {\n    if (!loadPromise) {\n      loadPromise = loadEvalFiles(patterns, conditions, baseCwd, runtime).catch((err) => {\n        loadPromise = undefined; // Allow retry on next request\n        throw err;\n      });\n    }\n    return loadPromise;\n  };\n}\n\n// ── Core loading logic ─────────────────────────────────────────────\n\nasync function loadEvalFiles(\n  patterns: string[],\n  conditions: string[],\n  cwd: string,\n  runtime: AxlRuntime,\n): Promise<void> {\n  if (conditions.length > 0) {\n    await registerConditions(conditions);\n  }\n\n  const files = resolvePatterns(patterns, cwd);\n\n  if (files.length === 0) {\n    console.warn(`[axl-studio] No eval files found matching: ${patterns.join(', ')}`);\n    return;\n  }\n\n  for (const file of files) {\n    try {\n      const mod = await importModule(file, parentURL);\n      const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        console.warn(\n          `[axl-studio] Skipping ${file}: not a valid eval config ` +\n            `(missing workflow, dataset, or scorers)`,\n        );\n        continue;\n      }\n\n      const name = deriveEvalName(file, cwd);\n\n      if (runtime.getRegisteredEval(name)) {\n        console.warn(\n          `[axl-studio] Eval name \"${name}\" from ${file} collides with an ` +\n            `already-registered eval — overwriting`,\n        );\n      }\n\n      runtime.registerEval(name, evalConfig, mod.executeWorkflow);\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      console.warn(`[axl-studio] Failed to load eval ${file}: ${msg}`);\n    }\n  }\n}\n\n// ── Internal helpers ───────────────────────────────────────────────\n\nfunction normalizeConfig(config: EvalLoaderConfig): {\n  patterns: string[];\n  conditions: string[];\n} {\n  if (typeof config === 'string') {\n    return { patterns: [config], conditions: [] };\n  }\n  if (Array.isArray(config)) {\n    return { patterns: config, conditions: [] };\n  }\n  const files = typeof config.files === 'string' ? [config.files] : config.files;\n  return { patterns: files, conditions: config.conditions ?? [] };\n}\n\n/**\n * Derive eval name from file path relative to cwd.\n *\n * Examples (cwd = `/project`):\n * - `/project/evals/suggestions.eval.ts` → `\"evals/suggestions\"`\n * - `/project/evals/api/accuracy.eval.ts` → `\"evals/api/accuracy\"`\n */\nfunction deriveEvalName(filePath: string, cwd: string): string {\n  const rel = relative(cwd, filePath);\n  // Normalize to forward slashes for cross-platform consistency\n  const normalized = rel.replace(/\\\\/g, '/');\n  // Guard: file outside cwd (symlink, absolute path) — fall back to basename\n  if (normalized.startsWith('../')) {\n    const base = basename(filePath);\n    const stripped = base.replace(/\\.eval\\.[mc]?[jt]sx?$/, '');\n    return stripped !== base ? stripped : base.replace(/\\.[mc]?[jt]sx?$/, '') || base;\n  }\n  // Strip .eval.ts, .eval.mjs, .eval.js, etc.\n  const withoutEval = normalized.replace(/\\.eval\\.[mc]?[jt]sx?$/, '');\n  if (withoutEval !== normalized) return withoutEval;\n  // Fallback: strip extension\n  const withoutExt = normalized.replace(/\\.[mc]?[jt]sx?$/, '');\n  return withoutExt || normalized;\n}\n\n/**\n * Resolve patterns to absolute file paths.\n *\n * Supports:\n * - Explicit file paths (no wildcards)\n * - Single-directory globs: `dir/*.eval.ts`\n * - Recursive globs: `dir/**\\/*.eval.ts` or `**\\/*.eval.ts`\n *\n * Multi-segment `**` (e.g., `a/**\\/b/**\\/*.ts`) is not supported.\n */\nfunction resolvePatterns(patterns: string[], cwd: string): string[] {\n  const files: string[] = [];\n  const seen = new Set<string>();\n  for (const pattern of patterns) {\n    const resolved = pattern.includes('*') ? expandGlob(pattern, cwd) : [resolve(cwd, pattern)];\n    for (const file of resolved) {\n      if (!seen.has(file)) {\n        seen.add(file);\n        files.push(file);\n      }\n    }\n  }\n  return files;\n}\n\n/**\n * Expand a glob pattern to matching file paths.\n *\n * Supported forms:\n * - `dir/*.eval.ts`     — match files in dir/\n * - `dir/**\\/*.eval.ts` — recursively match under dir/\n * - `**\\/*.eval.ts`     — recursively match under cwd\n */\nfunction expandGlob(pattern: string, cwd: string): string[] {\n  if (pattern.includes('**/')) {\n    const sepIdx = pattern.indexOf('**/');\n    const baseDir = resolve(cwd, pattern.slice(0, sepIdx) || '.');\n    const fileGlob = pattern.slice(sepIdx + 3) || '*';\n    return findFiles(baseDir, fileGlob, true);\n  }\n\n  const dir = resolve(cwd, dirname(pattern));\n  const fileGlob = basename(pattern);\n  return findFiles(dir, fileGlob, false);\n}\n\nconst MAX_DEPTH = 20;\n\nfunction findFiles(dir: string, fileGlob: string, recursive: boolean, depth = 0): string[] {\n  if (depth > MAX_DEPTH) return [];\n  const matcher = globToRegex(fileGlob);\n  const results: string[] = [];\n\n  try {\n    const entries = readdirSync(dir);\n    for (const entry of entries) {\n      const full = resolve(dir, entry);\n      try {\n        const stat = statSync(full);\n        if (stat.isFile() && matcher.test(entry)) {\n          results.push(full);\n        } else if (stat.isDirectory() && recursive) {\n          results.push(...findFiles(full, fileGlob, true, depth + 1));\n        }\n      } catch {\n        // Skip unreadable entries\n      }\n    }\n  } catch {\n    // Directory doesn't exist or unreadable\n  }\n\n  return results;\n}\n\n/** Convert a simple glob pattern (e.g., `*.eval.ts`) to a RegExp. */\nfunction globToRegex(glob: string): RegExp {\n  const escaped = glob.replace(/[.+?^${}()|[\\]\\\\]/g, '\\\\$&').replace(/\\*/g, '.*');\n  return new RegExp(`^${escaped}$`);\n}\n\nasync function registerConditions(conditions: string[]): Promise<void> {\n  try {\n    const nodeModule = await import('node:module');\n    const hookCode = [\n      `const extra = ${JSON.stringify(conditions)};`,\n      `export async function resolve(specifier, context, nextResolve) {`,\n      `  return nextResolve(specifier, {`,\n      `    ...context,`,\n      `    conditions: [...new Set([...context.conditions, ...extra])],`,\n      `  });`,\n      `}`,\n    ].join('\\n');\n    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n  } catch {\n    console.warn('[axl-studio] Warning: import conditions require Node.js 20.6+');\n  }\n}\n"],"mappings":";;;;;;;;;AAAA,SAAS,WAAAA,UAAS,WAAAC,gBAAe;AACjC,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAC9B,SAAS,0BAA0B;AACnC,SAAS,uBAAuB;;;ACJhC,SAAS,SAAS,UAAU,SAAS,gBAAgB;AACrD,SAAS,aAAa,gBAAgB;AACtC,SAAS,qBAAqB;AAO9B,IAAM,YACJ,YAAY,OAAO,cAAc,OAAO,eAAe,cAAc,aAAa,SAAS,EAAE;AA+CxF,SAAS,iBACd,QACA,SACA,KACqB;AACrB,MAAI;AACJ,QAAM,EAAE,UAAU,WAAW,IAAI,gBAAgB,MAAM;AACvD,QAAM,UAAU,OAAO,QAAQ,IAAI;AAEnC,SAAO,MAAM;AACX,QAAI,CAAC,aAAa;AAChB,oBAAc,cAAc,UAAU,YAAY,SAAS,OAAO,EAAE,MAAM,CAAC,QAAQ;AACjF,sBAAc;AACd,cAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AACF;AAIA,eAAe,cACb,UACA,YACA,KACA,SACe;AACf,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAEA,QAAM,QAAQ,gBAAgB,UAAU,GAAG;AAE3C,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,KAAK,8CAA8C,SAAS,KAAK,IAAI,CAAC,EAAE;AAChF;AAAA,EACF;AAEA,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,MAAM,MAAM,aAAa,MAAM,SAAS;AAC9C,YAAM,aAAa,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAExE,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,gBAAQ;AAAA,UACN,yBAAyB,IAAI;AAAA,QAE/B;AACA;AAAA,MACF;AAEA,YAAM,OAAO,eAAe,MAAM,GAAG;AAErC,UAAI,QAAQ,kBAAkB,IAAI,GAAG;AACnC,gBAAQ;AAAA,UACN,2BAA2B,IAAI,UAAU,IAAI;AAAA,QAE/C;AAAA,MACF;AAEA,cAAQ,aAAa,MAAM,YAAY,IAAI,eAAe;AAAA,IAC5D,SAAS,KAAK;AACZ,YAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAQ,KAAK,oCAAoC,IAAI,KAAK,GAAG,EAAE;AAAA,IACjE;AAAA,EACF;AACF;AAIA,SAAS,gBAAgB,QAGvB;AACA,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO,EAAE,UAAU,CAAC,MAAM,GAAG,YAAY,CAAC,EAAE;AAAA,EAC9C;AACA,MAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,WAAO,EAAE,UAAU,QAAQ,YAAY,CAAC,EAAE;AAAA,EAC5C;AACA,QAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,CAAC,OAAO,KAAK,IAAI,OAAO;AACzE,SAAO,EAAE,UAAU,OAAO,YAAY,OAAO,cAAc,CAAC,EAAE;AAChE;AASA,SAAS,eAAe,UAAkB,KAAqB;AAC7D,QAAM,MAAM,SAAS,KAAK,QAAQ;AAElC,QAAM,aAAa,IAAI,QAAQ,OAAO,GAAG;AAEzC,MAAI,WAAW,WAAW,KAAK,GAAG;AAChC,UAAM,OAAO,SAAS,QAAQ;AAC9B,UAAM,WAAW,KAAK,QAAQ,yBAAyB,EAAE;AACzD,WAAO,aAAa,OAAO,WAAW,KAAK,QAAQ,mBAAmB,EAAE,KAAK;AAAA,EAC/E;AAEA,QAAM,cAAc,WAAW,QAAQ,yBAAyB,EAAE;AAClE,MAAI,gBAAgB,WAAY,QAAO;AAEvC,QAAM,aAAa,WAAW,QAAQ,mBAAmB,EAAE;AAC3D,SAAO,cAAc;AACvB;AAYA,SAAS,gBAAgB,UAAoB,KAAuB;AAClE,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,WAAW,UAAU;AAC9B,UAAM,WAAW,QAAQ,SAAS,GAAG,IAAI,WAAW,SAAS,GAAG,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC;AAC1F,eAAW,QAAQ,UAAU;AAC3B,UAAI,CAAC,KAAK,IAAI,IAAI,GAAG;AACnB,aAAK,IAAI,IAAI;AACb,cAAM,KAAK,IAAI;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAUA,SAAS,WAAW,SAAiB,KAAuB;AAC1D,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS,QAAQ,QAAQ,KAAK;AACpC,UAAM,UAAU,QAAQ,KAAK,QAAQ,MAAM,GAAG,MAAM,KAAK,GAAG;AAC5D,UAAMC,YAAW,QAAQ,MAAM,SAAS,CAAC,KAAK;AAC9C,WAAO,UAAU,SAASA,WAAU,IAAI;AAAA,EAC1C;AAEA,QAAM,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;AACzC,QAAM,WAAW,SAAS,OAAO;AACjC,SAAO,UAAU,KAAK,UAAU,KAAK;AACvC;AAEA,IAAM,YAAY;AAElB,SAAS,UAAU,KAAa,UAAkB,WAAoB,QAAQ,GAAa;AACzF,MAAI,QAAQ,UAAW,QAAO,CAAC;AAC/B,QAAM,UAAU,YAAY,QAAQ;AACpC,QAAM,UAAoB,CAAC;AAE3B,MAAI;AACF,UAAM,UAAU,YAAY,GAAG;AAC/B,eAAW,SAAS,SAAS;AAC3B,YAAM,OAAO,QAAQ,KAAK,KAAK;AAC/B,UAAI;AACF,cAAM,OAAO,SAAS,IAAI;AAC1B,YAAI,KAAK,OAAO,KAAK,QAAQ,KAAK,KAAK,GAAG;AACxC,kBAAQ,KAAK,IAAI;AAAA,QACnB,WAAW,KAAK,YAAY,KAAK,WAAW;AAC1C,kBAAQ,KAAK,GAAG,UAAU,MAAM,UAAU,MAAM,QAAQ,CAAC,CAAC;AAAA,QAC5D;AAAA,MACF,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AAEA,SAAO;AACT;AAGA,SAAS,YAAY,MAAsB;AACzC,QAAM,UAAU,KAAK,QAAQ,sBAAsB,MAAM,EAAE,QAAQ,OAAO,IAAI;AAC9E,SAAO,IAAI,OAAO,IAAI,OAAO,GAAG;AAClC;AAEA,eAAe,mBAAmB,YAAqC;AACrE,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,+DAA+D;AAAA,EAC9E;AACF;;;AD9EO,SAAS,uBAAuB,SAAkC;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA,WAAW;AAAA,IACX;AAAA,EACF,IAAI;AAGJ,QAAM,WAAW,kBAAkB,QAAQ,QAAQ;AAGnD,QAAM,aAAa,cAAc,kBAAkB,IAAI;AAEvD,MAAI,eAAe,CAAC,YAAY;AAC9B,UAAM,MACJ,YAAY,YACX,OAAO,cAAc,cAAc,YAAYC,SAAQ,cAAc,YAAY,GAAG,CAAC;AACxF,YAAQ;AAAA,MACN,qEACKC,SAAQ,KAAK,QAAQ,CAAC;AAAA,IAE7B;AAAA,EACF;AAGA,QAAM,aAAa,QAAQ,QAAQ,iBAAiB,QAAQ,OAAO,OAAO,IAAI;AAE9E,QAAM,EAAE,KAAK,SAAS,eAAe,iBAAiB,iBAAiB,IAAI,aAAa;AAAA,IACtF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM;AAAA;AAAA,IACN;AAAA,IACA,YAAY,QAAQ;AAAA,EACtB,CAAC;AAKD,MAAI,kBAAkB;AACpB,YAAQ,UAAU,gBAAgB;AAAA,EACpC;AAGA,MAAI,QAAQ,IAAI,aAAa,gBAAgB,CAAC,eAAe;AAC3D,YAAQ;AAAA,MACN;AAAA,IAGF;AAAA,EACF;AAKA,QAAM,WAAW,mBAAmB,IAAI,OAAO;AAAA,IAC7C,uBAAuB;AAAA,EACzB,CAAC;AAED,MAAI,SAAS;AAEb,WAAS,QAAQ,KAAsB,KAAqB;AAC1D,QAAI,QAAQ;AACV,UAAI,UAAU,KAAK,EAAE,gBAAgB,mBAAmB,CAAC;AACzD,UAAI;AAAA,QACF,KAAK,UAAU;AAAA,UACb,IAAI;AAAA,UACJ,OAAO,EAAE,MAAM,UAAU,SAAS,uCAAuC;AAAA,QAC3E,CAAC;AAAA,MACH;AACA;AAAA,IACF;AAYA,UAAM,SAAS;AACf,QAAI,OAAO,QAAQ,QAAQ,CAAC,OAAO,SAAS;AAC1C,UAAI;AACF,eAAO,UAAU,OAAO,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;AAAA,MAC1D,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,aAAS,KAAK,GAAG,EAAE,MAAM,CAAC,QAAQ;AAChC,cAAQ,MAAM,oDAAoD,GAAG;AACrE,UAAI,CAAC,IAAI,aAAa;AACpB,YAAI,UAAU,KAAK,EAAE,gBAAgB,mBAAmB,CAAC;AACzD,YAAI;AAAA,UACF,KAAK,UAAU;AAAA,YACb,IAAI;AAAA,YACJ,OAAO,EAAE,MAAM,kBAAkB,SAAS,wBAAwB;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAMA,WAAS,gBAAgB,IAAqB,UAAoB;AAChE,QAAI,QAAQ;AACV,SAAG,MAAM;AACT;AAAA,IACF;AACA,UAAM,SAAS;AAAA,MACb,MAAM,CAAC,SAAiB,GAAG,KAAK,IAAI;AAAA,MACpC,OAAO,MAAM,GAAG,MAAM;AAAA,IACxB;AACA,YAAQ,IAAI,MAAM;AAClB,QAAI,aAAa,QAAW;AAC1B,cAAQ,YAAY,QAAQ,QAAQ;AAAA,IACtC;AAEA,OAAG,GAAG,WAAW,CAAC,QAAQ;AACxB,YAAM,QAAQ,gBAAgB,OAAO,GAAG,GAAG,QAAQ,OAAO;AAC1D,UAAI,MAAO,IAAG,KAAK,KAAK;AAAA,IAC1B,CAAC;AAED,OAAG,GAAG,SAAS,MAAM,QAAQ,OAAO,MAAM,CAAC;AAC3C,OAAG,GAAG,SAAS,MAAM,QAAQ,OAAO,MAAM,CAAC;AAAA,EAC7C;AAGA,MAAI;AAEJ,MAAI;AACJ,MAAI;AAGJ,WAAS,iBAAiB,QAAgB,MAAe;AACvD,QAAI,KAAK;AACP,YAAM,IAAI;AAAA,QACR;AAAA,MAEF;AAAA,IACF;AAEA,UAAM,SAAS,SAAS,WAAW,GAAG,QAAQ,QAAQ;AAEtD,UAAM,IAAI,gBAAgB,EAAE,UAAU,KAAK,CAAC;AAC5C,gBAAY;AAEZ,qBAAiB,OAAO,KAAsB,QAAa,SAAiB;AAE1E,YAAM,WAAW,IAAI,IAAI,IAAI,KAAM,UAAU,IAAI,QAAQ,IAAI,EAAE,EAAE;AACjE,UAAI,aAAa,OAAQ;AAKzB,UAAI,QAAQ;AACV,eAAO,QAAQ;AACf;AAAA,MACF;AAIA,UAAI;AACJ,UAAI,eAAe;AACjB,YAAI;AACF,gBAAM,SAAS,MAAM,cAAc,GAAG;AAEtC,gBAAM,UAAU,OAAO,WAAW,YAAY,SAAS,OAAO;AAC9D,cAAI,CAAC,SAAS;AACZ,mBAAO,MAAM,mCAAmC;AAChD,mBAAO,QAAQ;AACf;AAAA,UACF;AACA,cAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,iCAAqB,OAAO;AAAA,UAC9B;AAAA,QACF,QAAQ;AACN,iBAAO,MAAM,gCAAgC;AAC7C,iBAAO,QAAQ;AACf;AAAA,QACF;AAAA,MACF;AASA,UAAI,UAAU,CAAC,KAAK;AAClB,eAAO,QAAQ;AACf;AAAA,MACF;AAEA,UAAI,cAAc,KAAK,QAAQ,MAAM,CAAC,OAAO;AAC3C,wBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAAA,IACH;AAEA,WAAO,GAAG,WAAW,cAAc;AAAA,EACrC;AAGA,WAAS,QAAQ;AACf,aAAS;AAGT,oBAAgB;AAGhB,YAAQ,SAAS;AAGjB,QAAI,kBAAkB,WAAW;AAC/B,gBAAU,eAAe,WAAW,cAAc;AAClD,uBAAiB;AACjB,kBAAY;AAAA,IACd;AAGA,QAAI,KAAK;AACP,UAAI,MAAM;AACV,YAAM;AAAA,IACR;AAGA,QAAI,eAAe;AACjB,cAAQ,eAAe,SAAS,aAAa;AAAA,IAC/C;AAGA,qBAAiB;AAAA,EACnB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,mBAAmB;AAAA,IACnB;AAAA,EACF;AACF;AASA,SAAS,kBAAkB,KAAsB;AAC/C,MAAI,CAAC,IAAK,QAAO;AAGjB,QAAM,aAAa,IAAI,QAAQ,QAAQ,EAAE;AACzC,MAAI,CAAC,WAAY,QAAO;AAGxB,MAAI,CAAC,WAAW,WAAW,GAAG,GAAG;AAC/B,UAAM,IAAI,MAAM,sCAAsC,GAAG,wBAAwB;AAAA,EACnF;AAGA,MAAI,WAAW,SAAS,IAAI,GAAG;AAC7B,UAAM,IAAI,MAAM,iDAAiD,GAAG,IAAI;AAAA,EAC1E;AACA,MAAI,WAAW,SAAS,IAAI,GAAG;AAC7B,UAAM,IAAI,MAAM,uDAAuD,GAAG,IAAI;AAAA,EAChF;AACA,MAAI,CAAC,sBAAsB,KAAK,UAAU,GAAG;AAC3C,UAAM,IAAI;AAAA,MACR,8CAA8C,GAAG;AAAA,IAEnD;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAwC;AAE/C,QAAM,MACJ,YAAY,YACX,OAAO,cAAc,cAAc,YAAYD,SAAQ,cAAc,YAAY,GAAG,CAAC;AACxF,QAAM,YAAYC,SAAQ,KAAK,QAAQ;AACvC,SAAO,WAAWA,SAAQ,WAAW,YAAY,CAAC,IAAI,YAAY;AACpE;","names":["resolve","dirname","fileGlob","dirname","resolve"]}
+{"version":3,"sources":["../src/middleware.ts","../src/eval-loader.ts"],"sourcesContent":["import { resolve, dirname } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { fileURLToPath } from 'node:url';\nimport { getRequestListener } from '@hono/node-server';\nimport { WebSocketServer } from 'ws';\nimport { createServer } from './server/index.js';\nimport { handleWsMessage } from './server/ws/protocol.js';\nimport { createEvalLoader } from './eval-loader.js';\nimport type { EvalLoaderConfig } from './eval-loader.js';\nimport type { BufferCaps } from './server/ws/connection-manager.js';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport type { IncomingMessage, ServerResponse, Server } from 'node:http';\n\nexport type { EvalLoaderConfig } from './eval-loader.js';\nexport type { BufferCaps } from './server/ws/connection-manager.js';\n\nexport type StudioMiddlewareOptions = {\n  /** The AxlRuntime instance to observe and control. */\n  runtime: AxlRuntime;\n\n  /**\n   * URL path prefix where Studio is mounted.\n   * Must match the mount path in your framework (Express `app.use()`,\n   * Fastify `register()`, Hono `app.route()`, etc.). The framework is\n   * expected to strip the prefix from `req.url` before calling the handler.\n   *\n   * Do not set basePath when using a raw `http.Server` as the root handler —\n   * leave it empty and mount at root instead.\n   *\n   * Must start with '/' when non-empty. Trailing slashes are stripped.\n   * Only URL-safe characters allowed: [a-zA-Z0-9/_-]\n   *\n   * @example '/studio'\n   * @example '/admin/studio'\n   * @example ''  — mounted at root (default)\n   */\n  basePath?: string;\n\n  /**\n   * Serve the pre-built Studio SPA.\n   * Set to false if serving the client from a CDN or separate build.\n   * @default true\n   */\n  serveClient?: boolean;\n\n  /**\n   * Verify a WebSocket upgrade request before completing the handshake.\n   *\n   * Return either a boolean (`true` allow, `false` reject) or an object\n   * `{ allowed: boolean, metadata?: unknown }` to ALSO attach per-connection\n   * metadata that `filterTraceEvent` can later read — typically the auth\n   * subject (userId, tenantId, role). Throw to reject with an error.\n   *\n   * IMPORTANT: WebSocket upgrades bypass Express/Fastify/Koa middleware.\n   * If your HTTP routes are behind auth middleware, WS connections are NOT\n   * automatically protected. Use this callback to enforce authentication\n   * on WebSocket connections.\n   *\n   * @example Multi-tenant: extract tenant from JWT, attach as metadata\n   * verifyUpgrade: async (req) => {\n   *   const token = req.headers.authorization?.slice(7);\n   *   const claims = await verifyJwt(token);\n   *   if (!claims) return false;\n   *   return { allowed: true, metadata: { userId: claims.sub, tenantId: claims.tid } };\n   * }\n   */\n  verifyUpgrade?: (\n    req: IncomingMessage,\n  ) =>\n    | boolean\n    | { allowed: boolean; metadata?: unknown }\n    | Promise<boolean | { allowed: boolean; metadata?: unknown }>;\n\n  /**\n   * Optional per-event filter used by multi-tenant deployments to scope the\n   * trace firehose. Called on every outbound WebSocket broadcast; return\n   * `true` to deliver to this connection, `false` to drop.\n   *\n   * `event` is the parsed event payload — for the `trace:*` channel this is\n   * an `AxlEvent` from `@axlsdk/axl` (narrow via the discriminated union);\n   * for `costs` it's a `CostData`; for `execution:*` / `eval:*` it's the\n   * legacy stream-event shape (until PR 3 collapses the wire to `AxlEvent`).\n   * Typed `unknown` because the filter runs for every channel.\n   * `metadata` is the per-connection metadata attached by `verifyUpgrade`,\n   * or `undefined` if `verifyUpgrade` returned a bare boolean.\n   *\n   * Filter errors are treated as `drop` (fail-closed) so a buggy predicate\n   * cannot accidentally leak events cross-tenant.\n   *\n   * @example Scope trace events by tenant id stored on agent metadata\n   * import type { AxlEvent } from '@axlsdk/axl';\n   *\n   * filterTraceEvent: (event, meta) => {\n   *   const m = meta as { tenantId?: string } | undefined;\n   *   if (!m?.tenantId) return false;\n   *   const e = event as AxlEvent;\n   *   // Only narrow event.type === 'agent_call_end' events; pass structural\n   *   // events (workflow_start/end, cost updates) through unconditionally.\n   *   if (e.type !== 'agent_call_end') return true;\n   *   return e.workflow?.startsWith(`${m.tenantId}:`) ?? false;\n   * }\n   */\n  filterTraceEvent?: (event: unknown, metadata: unknown) => boolean;\n\n  /**\n   * Disable all mutating endpoints (execute, test, send, delete, resolve).\n   * When true, Studio is observation-only.\n   * @default false\n   */\n  readOnly?: boolean;\n\n  /**\n   * Lazy-load eval files for the Eval Runner panel.\n   *\n   * Eval files are dynamically imported on first access to eval endpoints,\n   * not at middleware construction time. This means:\n   * - Zero cost during normal API operation\n   * - Eval files can import from any module without creating circular deps\n   *   in the static module graph (they're loaded as standalone entry points)\n   * - `@axlsdk/eval` can remain a devDependency — eval files never enter\n   *   the production bundle since bundlers can't see dynamic imports\n   *\n   * Accepts glob patterns, explicit file paths, or an object with\n   * `conditions` for monorepo source export resolution.\n   *\n   * Eval files are loaded once and cached for the middleware's lifetime.\n   * Changes to eval files require a server restart.\n   *\n   * @example\n   * // Single glob pattern\n   * evals: 'evals/*.eval.ts'\n   *\n   * @example\n   * // Multiple patterns\n   * evals: ['libs/api/evals/*.eval.ts', 'libs/ai/evals/*.eval.ts']\n   *\n   * @example\n   * // With import conditions for monorepo source exports\n   * evals: {\n   *   files: 'libs/api/evals/*.eval.ts',\n   *   conditions: ['development'],\n   * }\n   */\n  evals?: EvalLoaderConfig;\n\n  /**\n   * Override the default WebSocket replay-buffer resource caps for\n   * production deployments under sustained execution churn.\n   *\n   * Defaults: `maxEventsPerBuffer: 1000`, `maxBytesPerBuffer: 4 MiB`,\n   * `maxActiveBuffers: 256`. Worst-case memory is roughly\n   * `maxActiveBuffers × maxBytesPerBuffer` (≈1 GiB at defaults). Tighten\n   * either dimension to lower the ceiling at the cost of late-subscriber\n   * replay coverage; raise them if you need more event headroom on long\n   * verbose-mode streams.\n   *\n   * Terminal `done` / `error` events are always buffered regardless of\n   * caps, so a late subscriber to a completed stream still sees the\n   * outcome — only intermediate non-terminal events are dropped.\n   *\n   * All fields are optional; passing `{}` is a no-op.\n   *\n   * @example\n   * // Halve memory ceiling on a high-churn deployment\n   * bufferCaps: { maxActiveBuffers: 128, maxBytesPerBuffer: 2 * 1024 * 1024 }\n   */\n  bufferCaps?: BufferCaps;\n};\n\n/**\n * Minimal contract a WebSocket connection must satisfy.\n * Matches the `ws` library API (de facto standard in Node.js).\n */\nexport interface StudioWebSocket {\n  send(data: string): void;\n  close(): void;\n  on(event: 'message', fn: (data: string | Buffer) => void): void;\n  on(event: 'close', fn: () => void): void;\n  on(event: 'error', fn: (err: Error) => void): void;\n}\n\n// Re-export for Hono-in-Hono consumers\nexport { handleWsMessage } from './server/ws/protocol.js';\n\nexport type StudioMiddleware = ReturnType<typeof createStudioMiddleware>;\n\nexport function createStudioMiddleware(options: StudioMiddlewareOptions) {\n  const {\n    runtime,\n    serveClient = true,\n    verifyUpgrade,\n    readOnly = false,\n    filterTraceEvent,\n  } = options;\n\n  // Normalize basePath: strip trailing slashes, validate format\n  const basePath = normalizeBasePath(options.basePath);\n\n  // Resolve pre-built SPA assets from this package's dist/\n  const staticRoot = serveClient ? resolveClientDist() : undefined;\n\n  if (serveClient && !staticRoot) {\n    const dir =\n      import.meta.dirname ??\n      (typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url)));\n    console.warn(\n      '[axl-studio] serveClient is true but no pre-built client found at ' +\n        `${resolve(dir, 'client')}. Studio UI will not be available. ` +\n        'Set serveClient: false to suppress this warning.',\n    );\n  }\n\n  // Create lazy eval loader if eval files are configured\n  const evalLoader = options.evals ? createEvalLoader(options.evals, runtime) : undefined;\n\n  const { app, connMgr, traceListener, closeActiveRuns, closeAggregators } = createServer({\n    runtime,\n    staticRoot,\n    basePath,\n    readOnly,\n    cors: false, // Host framework owns CORS policy\n    evalLoader,\n    bufferCaps: options.bufferCaps,\n  });\n\n  // Install the broadcast filter once at construction time. The connMgr\n  // applies it on every outbound event; `metadata` is attached to each\n  // connection after a successful `verifyUpgrade`.\n  if (filterTraceEvent) {\n    connMgr.setFilter(filterTraceEvent);\n  }\n\n  // Log production safety warning\n  if (process.env.NODE_ENV === 'production' && !verifyUpgrade) {\n    console.warn(\n      '[axl-studio] WARNING: Studio middleware mounted in production without verifyUpgrade. ' +\n        'WebSocket connections are not authenticated. All registered workflows, tools, and ' +\n        'agents are accessible. See https://axlsdk.com/docs/studio/security',\n    );\n  }\n\n  // Convert Hono app → Node.js (req, res) handler via @hono/node-server.\n  // overrideGlobalObjects: false prevents replacing global.Request/Response,\n  // which could break the host application.\n  const listener = getRequestListener(app.fetch, {\n    overrideGlobalObjects: false,\n  });\n\n  let closed = false;\n\n  function handler(req: IncomingMessage, res: ServerResponse) {\n    if (closed) {\n      res.writeHead(503, { 'Content-Type': 'application/json' });\n      res.end(\n        JSON.stringify({\n          ok: false,\n          error: { code: 'CLOSED', message: 'Studio middleware has been shut down' },\n        }),\n      );\n      return;\n    }\n\n    // Express/NestJS/Koa body parsers consume the raw IncomingMessage stream\n    // and store the parsed result on req.body. When that happens,\n    // @hono/node-server's getRequestListener reads an empty stream and Hono\n    // sees no body. To fix this, we re-serialize req.body as req.rawBody —\n    // a Buffer that getRequestListener checks before falling back to the\n    // stream (see newRequestFromIncoming in @hono/node-server/dist/listener.js).\n    //\n    // Verified against @hono/node-server@1.19.9. If this breaks after an\n    // upgrade, check whether the rawBody instanceof Buffer check still exists\n    // in newRequestFromIncoming.\n    const reqAny = req as unknown as Record<string, unknown>;\n    if (reqAny.body != null && !reqAny.rawBody) {\n      try {\n        reqAny.rawBody = Buffer.from(JSON.stringify(reqAny.body));\n      } catch {\n        // Non-serializable body — Hono will see an empty body\n      }\n    }\n\n    listener(req, res).catch((err) => {\n      console.error('[axl-studio] Unhandled error in request handler:', err);\n      if (!res.headersSent) {\n        res.writeHead(500, { 'Content-Type': 'application/json' });\n        res.end(\n          JSON.stringify({\n            ok: false,\n            error: { code: 'INTERNAL_ERROR', message: 'Internal server error' },\n          }),\n        );\n      }\n    });\n  }\n\n  // Handle an individual WebSocket using the Studio protocol.\n  // Adapts any StudioWebSocket to ConnectionManager's internal BroadcastTarget.\n  // `metadata` is whatever `verifyUpgrade` attached — passed through so the\n  // `filterTraceEvent` callback can read it on every outbound event.\n  function handleWebSocket(ws: StudioWebSocket, metadata?: unknown) {\n    if (closed) {\n      ws.close();\n      return;\n    }\n    const socket = {\n      send: (data: string) => ws.send(data),\n      close: () => ws.close(),\n    };\n    connMgr.add(socket);\n    if (metadata !== undefined) {\n      connMgr.setMetadata(socket, metadata);\n    }\n\n    ws.on('message', (raw) => {\n      const reply = handleWsMessage(String(raw), socket, connMgr);\n      if (reply) ws.send(reply);\n    });\n\n    ws.on('close', () => connMgr.remove(socket));\n    ws.on('error', () => connMgr.remove(socket));\n  }\n\n  // Internal WebSocketServer — created lazily by upgradeWebSocket()\n  let wss: InstanceType<typeof WebSocketServer> | undefined;\n  // References for cleanup: the upgrade handler and server it's attached to\n  let upgradeHandler: ((...args: any[]) => void) | undefined;\n  let serverRef: Server | undefined;\n\n  // Convenience: attach WS handling to an http.Server.\n  function upgradeWebSocket(server: Server, path?: string) {\n    if (wss) {\n      throw new Error(\n        '[axl-studio] upgradeWebSocket() has already been called. ' +\n          'Call close() first if you need to re-attach.',\n      );\n    }\n\n    const wsPath = path ?? (basePath ? `${basePath}/ws` : '/ws');\n\n    wss = new WebSocketServer({ noServer: true });\n    serverRef = server;\n\n    upgradeHandler = async (req: IncomingMessage, socket: any, head: Buffer) => {\n      // Match path, ignoring query string\n      const pathname = new URL(req.url!, `http://${req.headers.host}`).pathname;\n      if (pathname !== wsPath) return; // Let other upgrade handlers run\n\n      // Early-reject if close() already started. This is a cheap pre-check —\n      // we repeat it AFTER the async verifyUpgrade resolves to close the race\n      // where close() runs while verifyUpgrade is still in flight.\n      if (closed) {\n        socket.destroy();\n        return;\n      }\n\n      // Apply auth verification and capture optional per-connection metadata\n      // (e.g. { userId, tenantId }) that filterTraceEvent can later read.\n      let connectionMetadata: unknown;\n      if (verifyUpgrade) {\n        try {\n          const result = await verifyUpgrade(req);\n          // Normalize both return shapes: plain boolean or { allowed, metadata }\n          const allowed = typeof result === 'boolean' ? result : result.allowed;\n          if (!allowed) {\n            socket.write('HTTP/1.1 401 Unauthorized\\r\\n\\r\\n');\n            socket.destroy();\n            return;\n          }\n          if (typeof result === 'object' && result !== null) {\n            connectionMetadata = result.metadata;\n          }\n        } catch {\n          socket.write('HTTP/1.1 403 Forbidden\\r\\n\\r\\n');\n          socket.destroy();\n          return;\n        }\n      }\n\n      // Guard against close() having run during async verifyUpgrade. We check\n      // BOTH `closed` (set at the top of `close()`) and `!wss` (set at the\n      // end). `closed` is the authoritative signal — checking `!wss` alone\n      // wouldn't catch the window where `close()` has marked the middleware\n      // as shut down but hasn't yet torn down the WebSocketServer. Failing\n      // open here would leak a connection past the middleware's lifetime\n      // and leave it holding references the host expected to be released.\n      if (closed || !wss) {\n        socket.destroy();\n        return;\n      }\n\n      wss.handleUpgrade(req, socket, head, (ws) => {\n        handleWebSocket(ws, connectionMetadata);\n      });\n    };\n\n    server.on('upgrade', upgradeHandler);\n  }\n\n  // Cleanup function for lifecycle management.\n  function close() {\n    closed = true;\n\n    // Abort active streaming eval runs before closing connections\n    closeActiveRuns();\n\n    // Close all WebSocket connections\n    connMgr.closeAll();\n\n    // Remove the upgrade listener from the server before closing WSS\n    if (upgradeHandler && serverRef) {\n      serverRef.removeListener('upgrade', upgradeHandler);\n      upgradeHandler = undefined;\n      serverRef = undefined;\n    }\n\n    // Shut down the internal WebSocketServer if one was created\n    if (wss) {\n      wss.close();\n      wss = undefined;\n    }\n\n    // Remove only our trace event listener from the runtime\n    if (traceListener) {\n      runtime.removeListener('trace', traceListener);\n    }\n\n    // Close all aggregators (clear intervals and unsubscribe listeners)\n    closeAggregators();\n  }\n\n  return {\n    handler,\n    handleWebSocket,\n    upgradeWebSocket,\n    app,\n    connectionManager: connMgr,\n    close,\n  };\n}\n\n/**\n * Normalize and validate basePath.\n * - Empty string and undefined → ''\n * - Strip trailing slashes\n * - Validate leading slash when non-empty\n * - Reject unsafe characters\n */\nfunction normalizeBasePath(raw?: string): string {\n  if (!raw) return '';\n\n  // Strip trailing slashes\n  const normalized = raw.replace(/\\/+$/, '');\n  if (!normalized) return '';\n\n  // Must start with /\n  if (!normalized.startsWith('/')) {\n    throw new Error(`basePath must start with '/' (got '${raw}'). Example: '/studio'`);\n  }\n\n  // Reject path traversal, consecutive slashes, and unsafe characters\n  if (normalized.includes('..')) {\n    throw new Error(`basePath must not contain '..' segments (got '${raw}')`);\n  }\n  if (normalized.includes('//')) {\n    throw new Error(`basePath must not contain consecutive slashes (got '${raw}')`);\n  }\n  if (!/^\\/[a-zA-Z0-9/_-]*$/.test(normalized)) {\n    throw new Error(\n      `basePath contains invalid characters (got '${raw}'). ` +\n        'Only alphanumeric characters, /, _, and - are allowed.',\n    );\n  }\n\n  return normalized;\n}\n\nfunction resolveClientDist(): string | undefined {\n  // Resolve the directory of this file (dist/ in published package).\n  const dir =\n    import.meta.dirname ??\n    (typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url)));\n  const candidate = resolve(dir, 'client');\n  return existsSync(resolve(candidate, 'index.html')) ? candidate : undefined;\n}\n","import { resolve, relative, basename } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport type { AxlRuntime, EvalExecuteWorkflow } from '@axlsdk/axl';\nimport { importModule, expandGlob, registerConditions, pickDefault, pickExport } from '@axlsdk/axl';\n\n// In the CJS bundle, tsup stubs import.meta as {} so import.meta.url is\n// undefined. Fall back to __filename (which CJS defines) converted to a\n// file:// URL so tsImport() gets a valid parentURL.\nconst parentURL: string =\n  import.meta.url ?? pathToFileURL(typeof __filename !== 'undefined' ? __filename : __dirname).href;\n\n/**\n * Configuration for lazy eval file discovery.\n *\n * - `string` — a glob pattern or explicit file path\n * - `string[]` — multiple patterns/paths\n * - `object` — patterns with optional import conditions\n */\nexport type EvalLoaderConfig =\n  | string\n  | string[]\n  | {\n      files: string | string[];\n\n      /**\n       * Custom Node.js import conditions (e.g., `['development']`).\n       *\n       * In monorepos, package.json `exports` often use the `development` condition\n       * to point at source (`.ts`) instead of built dist. Without this, eval files\n       * that import workspace packages resolve to dist files, which may not exist.\n       *\n       * **WARNING**: Conditions are registered process-wide via `module.register()`.\n       * They affect all subsequent imports in the process, not just eval files.\n       */\n      conditions?: string[];\n    };\n\n/**\n * Create a lazy eval loader that resolves file patterns and dynamically imports\n * eval files on first call, registering them with the runtime.\n *\n * The loader is idempotent — subsequent calls return the same promise.\n * Concurrent callers all await the same loading work.\n *\n * Eval files should export a default config with `{ workflow, dataset, scorers }`\n * (the result of `defineEval()` from `@axlsdk/eval`). An optional named export\n * `executeWorkflow` overrides the default `runtime.execute()` behavior.\n *\n * Eval names are the file's path relative to `cwd` (project root), minus the\n * `.eval.*` suffix. This makes names completely stable — a file's name never\n * changes regardless of what other files or patterns exist.\n *\n * @param config  Glob patterns, file paths, or object with conditions\n * @param runtime The AxlRuntime to register discovered evals on\n * @param cwd     Base directory for resolving patterns and deriving names (default: `process.cwd()`)\n */\nexport function createEvalLoader(\n  config: EvalLoaderConfig,\n  runtime: AxlRuntime,\n  cwd?: string,\n): () => Promise<void> {\n  let loadPromise: Promise<void> | undefined;\n  const { patterns, conditions } = normalizeConfig(config);\n  const baseCwd = cwd ?? process.cwd();\n\n  return () => {\n    if (!loadPromise) {\n      loadPromise = loadEvalFiles(patterns, conditions, baseCwd, runtime).catch((err) => {\n        loadPromise = undefined; // Allow retry on next request\n        throw err;\n      });\n    }\n    return loadPromise;\n  };\n}\n\n// ── Core loading logic ─────────────────────────────────────────────\n\nasync function loadEvalFiles(\n  patterns: string[],\n  conditions: string[],\n  cwd: string,\n  runtime: AxlRuntime,\n): Promise<void> {\n  if (conditions.length > 0) {\n    await registerConditions(conditions);\n  }\n\n  const files = resolvePatterns(patterns, cwd);\n\n  if (files.length === 0) {\n    console.warn(`[axl-studio] No eval files found matching: ${patterns.join(', ')}`);\n    return;\n  }\n\n  for (const file of files) {\n    try {\n      const mod = await importModule(file, parentURL);\n      const evalConfig = pickDefault<{\n        workflow?: string;\n        dataset?: unknown;\n        scorers?: unknown;\n      }>(mod);\n\n      if (!evalConfig || typeof evalConfig !== 'object') {\n        console.warn(\n          `[axl-studio] Skipping ${file}: default export is ` +\n            `${evalConfig === null ? 'null' : typeof evalConfig}, expected an eval config object.`,\n        );\n        continue;\n      }\n      if (Array.isArray(evalConfig.scorers) && evalConfig.scorers.length === 0) {\n        console.warn(\n          `[axl-studio] Skipping ${file}: scorers array is empty — at least one scorer is required.`,\n        );\n        continue;\n      }\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        const missing = [\n          !evalConfig.workflow && 'workflow',\n          !evalConfig.dataset && 'dataset',\n          !evalConfig.scorers && 'scorers',\n        ]\n          .filter(Boolean)\n          .join(', ');\n        const keys = Object.keys(evalConfig).slice(0, 10);\n        const got = keys.length > 0 ? ` Got: { ${keys.join(', ')} }.` : '';\n        console.warn(\n          `[axl-studio] Skipping ${file}: not a valid eval config (missing ${missing}).${got}`,\n        );\n        continue;\n      }\n\n      const name = deriveEvalName(file, cwd);\n\n      if (runtime.getRegisteredEval(name)) {\n        console.warn(\n          `[axl-studio] Eval name \"${name}\" from ${file} collides with an ` +\n            `already-registered eval — overwriting`,\n        );\n      }\n\n      // Walk the ESM/CJS interop chain symmetrically with the default export\n      // so `executeWorkflow` stays visible when tsx loads the eval module as CJS\n      // (e.g. a `.ts` file in a package without `\"type\": \"module\"`). Validate\n      // the export is callable — a non-function `executeWorkflow` is silently\n      // ignored so registration falls back to `runtime.execute()`, with a\n      // warning so the user knows their export was rejected.\n      const exported = pickExport<unknown>(mod, 'executeWorkflow');\n      let customExecute: EvalExecuteWorkflow | undefined;\n      if (typeof exported === 'function') {\n        customExecute = exported as EvalExecuteWorkflow;\n      } else if (exported !== undefined) {\n        console.warn(\n          `[axl-studio] ${file} exports executeWorkflow but it is ${typeof exported}, ` +\n            `not a function — ignoring and falling back to runtime.execute()`,\n        );\n      }\n      runtime.registerEval(name, evalConfig, customExecute);\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      console.warn(`[axl-studio] Failed to load eval ${file}: ${msg}`);\n    }\n  }\n}\n\n// ── Internal helpers ───────────────────────────────────────────────\n\nfunction normalizeConfig(config: EvalLoaderConfig): {\n  patterns: string[];\n  conditions: string[];\n} {\n  if (typeof config === 'string') {\n    return { patterns: [config], conditions: [] };\n  }\n  if (Array.isArray(config)) {\n    return { patterns: config, conditions: [] };\n  }\n  const files = typeof config.files === 'string' ? [config.files] : config.files;\n  return { patterns: files, conditions: config.conditions ?? [] };\n}\n\n/**\n * Derive eval name from file path relative to cwd.\n *\n * Examples (cwd = `/project`):\n * - `/project/evals/suggestions.eval.ts` → `\"evals/suggestions\"`\n * - `/project/evals/api/accuracy.eval.ts` → `\"evals/api/accuracy\"`\n */\nfunction deriveEvalName(filePath: string, cwd: string): string {\n  const rel = relative(cwd, filePath);\n  // Normalize to forward slashes for cross-platform consistency\n  const normalized = rel.replace(/\\\\/g, '/');\n  // Guard: file outside cwd (symlink, absolute path) — fall back to basename\n  if (normalized.startsWith('../')) {\n    const base = basename(filePath);\n    const stripped = base.replace(/\\.eval\\.[mc]?[jt]sx?$/, '');\n    return stripped !== base ? stripped : base.replace(/\\.[mc]?[jt]sx?$/, '') || base;\n  }\n  // Strip .eval.ts, .eval.mjs, .eval.js, etc.\n  const withoutEval = normalized.replace(/\\.eval\\.[mc]?[jt]sx?$/, '');\n  if (withoutEval !== normalized) return withoutEval;\n  // Fallback: strip extension\n  const withoutExt = normalized.replace(/\\.[mc]?[jt]sx?$/, '');\n  return withoutExt || normalized;\n}\n\n/**\n * Resolve patterns to absolute file paths.\n *\n * Supports:\n * - Explicit file paths (no wildcards)\n * - Single-directory globs: `dir/*.eval.ts`\n * - Recursive globs: `dir/**\\/*.eval.ts` or `**\\/*.eval.ts`\n *\n * Multi-segment `**` (e.g., `a/**\\/b/**\\/*.ts`) is not supported.\n */\nfunction resolvePatterns(patterns: string[], cwd: string): string[] {\n  const files: string[] = [];\n  const seen = new Set<string>();\n  for (const pattern of patterns) {\n    const resolved = pattern.includes('*') ? expandGlob(pattern, cwd) : [resolve(cwd, pattern)];\n    for (const file of resolved) {\n      if (!seen.has(file)) {\n        seen.add(file);\n        files.push(file);\n      }\n    }\n  }\n  return files;\n}\n\n// `expandGlob` and `registerConditions` are imported from @axlsdk/axl above.\n// The studio-specific glob helpers and conditions registration that used to\n// live here are now shared with @axlsdk/eval to prevent drift.\n"],"mappings":";;;;;;AAAA,SAAS,WAAAA,UAAS,eAAe;AACjC,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAC9B,SAAS,0BAA0B;AACnC,SAAS,uBAAuB;;;ACJhC,SAAS,SAAS,UAAU,gBAAgB;AAC5C,SAAS,qBAAqB;AAE9B,SAAS,cAAc,YAAY,oBAAoB,aAAa,kBAAkB;AAKtF,IAAM,YACJ,YAAY,OAAO,cAAc,OAAO,eAAe,cAAc,aAAa,SAAS,EAAE;AA+CxF,SAAS,iBACd,QACA,SACA,KACqB;AACrB,MAAI;AACJ,QAAM,EAAE,UAAU,WAAW,IAAI,gBAAgB,MAAM;AACvD,QAAM,UAAU,OAAO,QAAQ,IAAI;AAEnC,SAAO,MAAM;AACX,QAAI,CAAC,aAAa;AAChB,oBAAc,cAAc,UAAU,YAAY,SAAS,OAAO,EAAE,MAAM,CAAC,QAAQ;AACjF,sBAAc;AACd,cAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AACF;AAIA,eAAe,cACb,UACA,YACA,KACA,SACe;AACf,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAEA,QAAM,QAAQ,gBAAgB,UAAU,GAAG;AAE3C,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,KAAK,8CAA8C,SAAS,KAAK,IAAI,CAAC,EAAE;AAChF;AAAA,EACF;AAEA,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,MAAM,MAAM,aAAa,MAAM,SAAS;AAC9C,YAAM,aAAa,YAIhB,GAAG;AAEN,UAAI,CAAC,cAAc,OAAO,eAAe,UAAU;AACjD,gBAAQ;AAAA,UACN,yBAAyB,IAAI,uBACxB,eAAe,OAAO,SAAS,OAAO,UAAU;AAAA,QACvD;AACA;AAAA,MACF;AACA,UAAI,MAAM,QAAQ,WAAW,OAAO,KAAK,WAAW,QAAQ,WAAW,GAAG;AACxE,gBAAQ;AAAA,UACN,yBAAyB,IAAI;AAAA,QAC/B;AACA;AAAA,MACF;AACA,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,cAAM,UAAU;AAAA,UACd,CAAC,WAAW,YAAY;AAAA,UACxB,CAAC,WAAW,WAAW;AAAA,UACvB,CAAC,WAAW,WAAW;AAAA,QACzB,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AACZ,cAAM,OAAO,OAAO,KAAK,UAAU,EAAE,MAAM,GAAG,EAAE;AAChD,cAAM,MAAM,KAAK,SAAS,IAAI,WAAW,KAAK,KAAK,IAAI,CAAC,QAAQ;AAChE,gBAAQ;AAAA,UACN,yBAAyB,IAAI,sCAAsC,OAAO,KAAK,GAAG;AAAA,QACpF;AACA;AAAA,MACF;AAEA,YAAM,OAAO,eAAe,MAAM,GAAG;AAErC,UAAI,QAAQ,kBAAkB,IAAI,GAAG;AACnC,gBAAQ;AAAA,UACN,2BAA2B,IAAI,UAAU,IAAI;AAAA,QAE/C;AAAA,MACF;AAQA,YAAM,WAAW,WAAoB,KAAK,iBAAiB;AAC3D,UAAI;AACJ,UAAI,OAAO,aAAa,YAAY;AAClC,wBAAgB;AAAA,MAClB,WAAW,aAAa,QAAW;AACjC,gBAAQ;AAAA,UACN,gBAAgB,IAAI,sCAAsC,OAAO,QAAQ;AAAA,QAE3E;AAAA,MACF;AACA,cAAQ,aAAa,MAAM,YAAY,aAAa;AAAA,IACtD,SAAS,KAAK;AACZ,YAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAQ,KAAK,oCAAoC,IAAI,KAAK,GAAG,EAAE;AAAA,IACjE;AAAA,EACF;AACF;AAIA,SAAS,gBAAgB,QAGvB;AACA,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO,EAAE,UAAU,CAAC,MAAM,GAAG,YAAY,CAAC,EAAE;AAAA,EAC9C;AACA,MAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,WAAO,EAAE,UAAU,QAAQ,YAAY,CAAC,EAAE;AAAA,EAC5C;AACA,QAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,CAAC,OAAO,KAAK,IAAI,OAAO;AACzE,SAAO,EAAE,UAAU,OAAO,YAAY,OAAO,cAAc,CAAC,EAAE;AAChE;AASA,SAAS,eAAe,UAAkB,KAAqB;AAC7D,QAAM,MAAM,SAAS,KAAK,QAAQ;AAElC,QAAM,aAAa,IAAI,QAAQ,OAAO,GAAG;AAEzC,MAAI,WAAW,WAAW,KAAK,GAAG;AAChC,UAAM,OAAO,SAAS,QAAQ;AAC9B,UAAM,WAAW,KAAK,QAAQ,yBAAyB,EAAE;AACzD,WAAO,aAAa,OAAO,WAAW,KAAK,QAAQ,mBAAmB,EAAE,KAAK;AAAA,EAC/E;AAEA,QAAM,cAAc,WAAW,QAAQ,yBAAyB,EAAE;AAClE,MAAI,gBAAgB,WAAY,QAAO;AAEvC,QAAM,aAAa,WAAW,QAAQ,mBAAmB,EAAE;AAC3D,SAAO,cAAc;AACvB;AAYA,SAAS,gBAAgB,UAAoB,KAAuB;AAClE,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,WAAW,UAAU;AAC9B,UAAM,WAAW,QAAQ,SAAS,GAAG,IAAI,WAAW,SAAS,GAAG,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC;AAC1F,eAAW,QAAQ,UAAU;AAC3B,UAAI,CAAC,KAAK,IAAI,IAAI,GAAG;AACnB,aAAK,IAAI,IAAI;AACb,cAAM,KAAK,IAAI;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;;;AD5CO,SAAS,uBAAuB,SAAkC;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA,WAAW;AAAA,IACX;AAAA,EACF,IAAI;AAGJ,QAAM,WAAW,kBAAkB,QAAQ,QAAQ;AAGnD,QAAM,aAAa,cAAc,kBAAkB,IAAI;AAEvD,MAAI,eAAe,CAAC,YAAY;AAC9B,UAAM,MACJ,YAAY,YACX,OAAO,cAAc,cAAc,YAAY,QAAQ,cAAc,YAAY,GAAG,CAAC;AACxF,YAAQ;AAAA,MACN,qEACKC,SAAQ,KAAK,QAAQ,CAAC;AAAA,IAE7B;AAAA,EACF;AAGA,QAAM,aAAa,QAAQ,QAAQ,iBAAiB,QAAQ,OAAO,OAAO,IAAI;AAE9E,QAAM,EAAE,KAAK,SAAS,eAAe,iBAAiB,iBAAiB,IAAI,aAAa;AAAA,IACtF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM;AAAA;AAAA,IACN;AAAA,IACA,YAAY,QAAQ;AAAA,EACtB,CAAC;AAKD,MAAI,kBAAkB;AACpB,YAAQ,UAAU,gBAAgB;AAAA,EACpC;AAGA,MAAI,QAAQ,IAAI,aAAa,gBAAgB,CAAC,eAAe;AAC3D,YAAQ;AAAA,MACN;AAAA,IAGF;AAAA,EACF;AAKA,QAAM,WAAW,mBAAmB,IAAI,OAAO;AAAA,IAC7C,uBAAuB;AAAA,EACzB,CAAC;AAED,MAAI,SAAS;AAEb,WAAS,QAAQ,KAAsB,KAAqB;AAC1D,QAAI,QAAQ;AACV,UAAI,UAAU,KAAK,EAAE,gBAAgB,mBAAmB,CAAC;AACzD,UAAI;AAAA,QACF,KAAK,UAAU;AAAA,UACb,IAAI;AAAA,UACJ,OAAO,EAAE,MAAM,UAAU,SAAS,uCAAuC;AAAA,QAC3E,CAAC;AAAA,MACH;AACA;AAAA,IACF;AAYA,UAAM,SAAS;AACf,QAAI,OAAO,QAAQ,QAAQ,CAAC,OAAO,SAAS;AAC1C,UAAI;AACF,eAAO,UAAU,OAAO,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;AAAA,MAC1D,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,aAAS,KAAK,GAAG,EAAE,MAAM,CAAC,QAAQ;AAChC,cAAQ,MAAM,oDAAoD,GAAG;AACrE,UAAI,CAAC,IAAI,aAAa;AACpB,YAAI,UAAU,KAAK,EAAE,gBAAgB,mBAAmB,CAAC;AACzD,YAAI;AAAA,UACF,KAAK,UAAU;AAAA,YACb,IAAI;AAAA,YACJ,OAAO,EAAE,MAAM,kBAAkB,SAAS,wBAAwB;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAMA,WAAS,gBAAgB,IAAqB,UAAoB;AAChE,QAAI,QAAQ;AACV,SAAG,MAAM;AACT;AAAA,IACF;AACA,UAAM,SAAS;AAAA,MACb,MAAM,CAAC,SAAiB,GAAG,KAAK,IAAI;AAAA,MACpC,OAAO,MAAM,GAAG,MAAM;AAAA,IACxB;AACA,YAAQ,IAAI,MAAM;AAClB,QAAI,aAAa,QAAW;AAC1B,cAAQ,YAAY,QAAQ,QAAQ;AAAA,IACtC;AAEA,OAAG,GAAG,WAAW,CAAC,QAAQ;AACxB,YAAM,QAAQ,gBAAgB,OAAO,GAAG,GAAG,QAAQ,OAAO;AAC1D,UAAI,MAAO,IAAG,KAAK,KAAK;AAAA,IAC1B,CAAC;AAED,OAAG,GAAG,SAAS,MAAM,QAAQ,OAAO,MAAM,CAAC;AAC3C,OAAG,GAAG,SAAS,MAAM,QAAQ,OAAO,MAAM,CAAC;AAAA,EAC7C;AAGA,MAAI;AAEJ,MAAI;AACJ,MAAI;AAGJ,WAAS,iBAAiB,QAAgB,MAAe;AACvD,QAAI,KAAK;AACP,YAAM,IAAI;AAAA,QACR;AAAA,MAEF;AAAA,IACF;AAEA,UAAM,SAAS,SAAS,WAAW,GAAG,QAAQ,QAAQ;AAEtD,UAAM,IAAI,gBAAgB,EAAE,UAAU,KAAK,CAAC;AAC5C,gBAAY;AAEZ,qBAAiB,OAAO,KAAsB,QAAa,SAAiB;AAE1E,YAAM,WAAW,IAAI,IAAI,IAAI,KAAM,UAAU,IAAI,QAAQ,IAAI,EAAE,EAAE;AACjE,UAAI,aAAa,OAAQ;AAKzB,UAAI,QAAQ;AACV,eAAO,QAAQ;AACf;AAAA,MACF;AAIA,UAAI;AACJ,UAAI,eAAe;AACjB,YAAI;AACF,gBAAM,SAAS,MAAM,cAAc,GAAG;AAEtC,gBAAM,UAAU,OAAO,WAAW,YAAY,SAAS,OAAO;AAC9D,cAAI,CAAC,SAAS;AACZ,mBAAO,MAAM,mCAAmC;AAChD,mBAAO,QAAQ;AACf;AAAA,UACF;AACA,cAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,iCAAqB,OAAO;AAAA,UAC9B;AAAA,QACF,QAAQ;AACN,iBAAO,MAAM,gCAAgC;AAC7C,iBAAO,QAAQ;AACf;AAAA,QACF;AAAA,MACF;AASA,UAAI,UAAU,CAAC,KAAK;AAClB,eAAO,QAAQ;AACf;AAAA,MACF;AAEA,UAAI,cAAc,KAAK,QAAQ,MAAM,CAAC,OAAO;AAC3C,wBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAAA,IACH;AAEA,WAAO,GAAG,WAAW,cAAc;AAAA,EACrC;AAGA,WAAS,QAAQ;AACf,aAAS;AAGT,oBAAgB;AAGhB,YAAQ,SAAS;AAGjB,QAAI,kBAAkB,WAAW;AAC/B,gBAAU,eAAe,WAAW,cAAc;AAClD,uBAAiB;AACjB,kBAAY;AAAA,IACd;AAGA,QAAI,KAAK;AACP,UAAI,MAAM;AACV,YAAM;AAAA,IACR;AAGA,QAAI,eAAe;AACjB,cAAQ,eAAe,SAAS,aAAa;AAAA,IAC/C;AAGA,qBAAiB;AAAA,EACnB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,mBAAmB;AAAA,IACnB;AAAA,EACF;AACF;AASA,SAAS,kBAAkB,KAAsB;AAC/C,MAAI,CAAC,IAAK,QAAO;AAGjB,QAAM,aAAa,IAAI,QAAQ,QAAQ,EAAE;AACzC,MAAI,CAAC,WAAY,QAAO;AAGxB,MAAI,CAAC,WAAW,WAAW,GAAG,GAAG;AAC/B,UAAM,IAAI,MAAM,sCAAsC,GAAG,wBAAwB;AAAA,EACnF;AAGA,MAAI,WAAW,SAAS,IAAI,GAAG;AAC7B,UAAM,IAAI,MAAM,iDAAiD,GAAG,IAAI;AAAA,EAC1E;AACA,MAAI,WAAW,SAAS,IAAI,GAAG;AAC7B,UAAM,IAAI,MAAM,uDAAuD,GAAG,IAAI;AAAA,EAChF;AACA,MAAI,CAAC,sBAAsB,KAAK,UAAU,GAAG;AAC3C,UAAM,IAAI;AAAA,MACR,8CAA8C,GAAG;AAAA,IAEnD;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAwC;AAE/C,QAAM,MACJ,YAAY,YACX,OAAO,cAAc,cAAc,YAAY,QAAQ,cAAc,YAAY,GAAG,CAAC;AACxF,QAAM,YAAYA,SAAQ,KAAK,QAAQ;AACvC,SAAO,WAAWA,SAAQ,WAAW,YAAY,CAAC,IAAI,YAAY;AACpE;","names":["resolve","resolve"]}

package/dist/server/index.cjs

@@ -138,8 +138,11 @@ function redactEvalItem(item) {
 }
 function redactEvalResult(result, redact) {
   if (!redact) return result;
+  const meta = result.metadata;
+  const scrubbedMetadata = meta && typeof meta.batchFailure === "string" ? { ...meta, batchFailure: REDACTED } : result.metadata;
   return {
     ...result,
+    metadata: scrubbedMetadata,
     items: result.items.map(redactEvalItem)
   };
 }
@@ -934,13 +937,18 @@ function reduceEvalTrends(acc, entry) {
   const cost = extractCost(entry.data);
   const model = extractModel(entry.data);
   const duration = extractDuration(entry.data);
+  const metadata = entry.data?.metadata;
+  const runGroupId = typeof metadata?.runGroupId === "string" ? metadata.runGroupId : void 0;
+  const batchAttempted = typeof metadata?.batchAttempted === "number" && Number.isFinite(metadata.batchAttempted) ? metadata.batchAttempted : void 0;
   const run = {
     timestamp: entry.timestamp,
     id: entry.id,
     scores,
     cost,
     ...model !== void 0 ? { model } : {},
-    ...duration !== void 0 ? { duration } : {}
+    ...duration !== void 0 ? { duration } : {},
+    ...runGroupId !== void 0 ? { runGroupId } : {},
+    ...batchAttempted !== void 0 ? { batchAttempted } : {}
   };
   const byEval = { ...acc.byEval };
   const prev = byEval[entry.eval];
@@ -1623,33 +1631,79 @@ function createEvalRoutes(connMgr, evalLoader) {
           if (runs > 1) {
             const runGroupId = (0, import_node_crypto.randomUUID)();
             const results = [];
+            let runFailure;
+            let cancelled = false;
             for (let r = 0; r < runs; r++) {
-              if (ac.signal.aborted) break;
-              const result = await runtime.runRegisteredEval(name, {
-                metadata: { runGroupId, runIndex: r },
-                signal: ac.signal,
-                captureTraces,
-                onProgress: (event) => {
-                  if (event.type === "run_done") return;
+              if (ac.signal.aborted) {
+                cancelled = true;
+                break;
+              }
+              try {
+                const result = await runtime.runRegisteredEval(name, {
+                  metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+                  signal: ac.signal,
+                  captureTraces,
+                  onProgress: (event) => {
+                    if (event.type === "run_done") return;
+                    connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                      ...event,
+                      run: r + 1,
+                      totalRuns: runs
+                    });
+                  }
+                });
+                results.push(result);
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_done",
+                  run: r + 1,
+                  totalRuns: runs
+                });
+              } catch (err) {
+                const isAbort = ac.signal.aborted || err instanceof Error && err.name === "AbortError";
+                if (isAbort) {
+                  cancelled = true;
                   connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                    ...event,
+                    type: "run_cancelled",
                     run: r + 1,
                     totalRuns: runs
                   });
+                  break;
                 }
-              });
-              results.push(result);
-              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                type: "run_done",
-                run: r + 1,
-                totalRuns: runs
-              });
+                runFailure = err instanceof Error ? err : new Error(String(err));
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_failed",
+                  run: r + 1,
+                  totalRuns: runs,
+                  message: redactErrorMessage(runFailure, redactOn)
+                });
+                break;
+              }
             }
             if (results.length > 0) {
+              const partial = results.length < runs;
+              const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
                 type: "done",
                 evalResultId: results[0].id,
-                runGroupId
+                runGroupId,
+                ...partial && {
+                  partial: true,
+                  batchCompleted: results.length,
+                  batchAttempted: runs,
+                  // `cancelled` and `batchFailure` are mutually exclusive:
+                  // the catch block sets at most one of {cancelled,
+                  // runFailure}. The client uses `cancelled` to render a
+                  // neutral "Cancelled — X of N runs completed" caption
+                  // instead of the amber "Stopped after: <message>"
+                  // failure caption.
+                  ...cancelled ? { cancelled: true } : {},
+                  ...failureMsg ? { batchFailure: failureMsg } : {}
+                }
+              });
+            } else if (runFailure) {
+              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                type: "error",
+                message: redactErrorMessage(runFailure, redactOn)
               });
             } else {
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
@@ -1687,19 +1741,38 @@ function createEvalRoutes(connMgr, evalLoader) {
         const { aggregateRuns } = await import("@axlsdk/eval");
         const runGroupId = (0, import_node_crypto.randomUUID)();
         const results = [];
+        let runFailure;
         for (let r = 0; r < runs; r++) {
-          const result2 = await runtime.runRegisteredEval(name, {
-            metadata: { runGroupId, runIndex: r },
-            captureTraces
-          });
-          results.push(result2);
+          try {
+            const result2 = await runtime.runRegisteredEval(name, {
+              metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+              captureTraces
+            });
+            results.push(result2);
+          } catch (err) {
+            runFailure = err instanceof Error ? err : new Error(String(err));
+            break;
+          }
+        }
+        if (results.length === 0) {
+          throw runFailure ?? new Error("No runs completed");
         }
-        const typedResults = results;
-        const aggregate = aggregateRuns(typedResults);
-        const first = typedResults[0];
+        const aggregate = aggregateRuns(results);
+        const first = results[0];
+        const partial = results.length < runs;
+        const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
         const result = {
           ...first,
-          _multiRun: { aggregate, allRuns: typedResults }
+          _multiRun: {
+            aggregate,
+            allRuns: results,
+            ...partial && {
+              partial: true,
+              batchCompleted: results.length,
+              batchAttempted: runs,
+              ...failureMsg ? { batchFailure: failureMsg } : {}
+            }
+          }
         };
         return c.json({
           ok: true,
@@ -1871,60 +1944,75 @@ function createEvalRoutes(connMgr, evalLoader) {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     const bad = (message) => c.json({ ok: false, error: { code: "BAD_REQUEST", message } }, 400);
-    if (!body.result || typeof body.result !== "object") {
+    if (body.result === void 0 || body.result === null) {
       return bad("result is required");
     }
-    const result = body.result;
-    if (!Array.isArray(result.items)) {
-      return bad("result.items must be an array");
-    }
-    if (typeof result.summary !== "object" || result.summary == null) {
-      return bad("result.summary must be an object");
-    }
-    if (typeof result.dataset !== "string" || !result.dataset) {
-      return bad("result.dataset must be a non-empty string (required for compare)");
+    const resultsRaw = Array.isArray(body.result) ? body.result : [body.result];
+    if (resultsRaw.length === 0) {
+      return bad("result must be a non-empty array or object");
     }
-    const summary = result.summary;
-    if (typeof summary.scorers !== "object" || summary.scorers == null) {
-      return bad("result.summary.scorers must be an object");
-    }
-    const summaryScorerNames = Object.keys(summary.scorers);
-    const items = result.items;
-    const summaryScorerSet = new Set(summaryScorerNames);
-    const uncoveredAcrossItems = /* @__PURE__ */ new Set();
-    for (const item of items) {
-      const itemScores = item?.scores;
-      if (itemScores && typeof itemScores === "object") {
-        for (const name of Object.keys(itemScores)) {
-          if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+    const validatedResults = [];
+    for (let i = 0; i < resultsRaw.length; i++) {
+      const entry = resultsRaw[i];
+      const prefix = resultsRaw.length > 1 ? `result[${i}]` : "result";
+      if (!entry || typeof entry !== "object") {
+        return bad(`${prefix} must be an object`);
+      }
+      const r = entry;
+      if (!Array.isArray(r.items)) {
+        return bad(`${prefix}.items must be an array`);
+      }
+      if (typeof r.summary !== "object" || r.summary == null) {
+        return bad(`${prefix}.summary must be an object`);
+      }
+      if (typeof r.dataset !== "string" || !r.dataset) {
+        return bad(`${prefix}.dataset must be a non-empty string (required for compare)`);
+      }
+      const summary = r.summary;
+      if (typeof summary.scorers !== "object" || summary.scorers == null) {
+        return bad(`${prefix}.summary.scorers must be an object`);
+      }
+      const summaryScorerNames = Object.keys(summary.scorers);
+      const items = r.items;
+      const summaryScorerSet = new Set(summaryScorerNames);
+      const uncoveredAcrossItems = /* @__PURE__ */ new Set();
+      for (const item of items) {
+        const itemScores = item?.scores;
+        if (itemScores && typeof itemScores === "object") {
+          for (const name of Object.keys(itemScores)) {
+            if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+          }
         }
       }
-    }
-    if (uncoveredAcrossItems.size > 0) {
-      return bad(
-        `item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
-      );
+      if (uncoveredAcrossItems.size > 0) {
+        return bad(
+          `${prefix} item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
+        );
+      }
+      validatedResults.push(r);
     }
     const trim = (v) => typeof v === "string" && v.trim() !== "" ? v.trim() : void 0;
-    const metadataObj = typeof result.metadata === "object" && result.metadata != null ? result.metadata : {};
+    const firstResult = validatedResults[0];
+    const metadataObj = typeof firstResult.metadata === "object" && firstResult.metadata != null ? firstResult.metadata : {};
     const workflowsFromMeta = Array.isArray(metadataObj.workflows) ? metadataObj.workflows : [];
     const primaryWorkflow = workflowsFromMeta.find((w) => typeof w === "string");
-    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? // Legacy fallback: pre-0.14 CLI artifacts had workflow at the top level.
-    trim(result.workflow) ?? "imported";
-    const id = (0, import_node_crypto.randomUUID)();
+    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? trim(firstResult.workflow) ?? "imported";
     const timestamp = Date.now();
-    const imported = {
-      ...result,
-      id,
-      metadata: typeof result.metadata === "object" && result.metadata != null ? result.metadata : {}
-    };
-    await runtime.saveEvalResult({
-      id,
-      eval: evalName,
-      timestamp,
-      data: imported
-    });
-    return c.json({ ok: true, data: { id, eval: evalName, timestamp } });
+    const imported = [];
+    for (const r of validatedResults) {
+      const id = (0, import_node_crypto.randomUUID)();
+      const entry = {
+        ...r,
+        id,
+        metadata: typeof r.metadata === "object" && r.metadata != null ? r.metadata : {}
+      };
+      await runtime.saveEvalResult({ id, eval: evalName, timestamp, data: entry });
+      imported.push({ id, eval: evalName, timestamp });
+    }
+    if (imported.length === 1) {
+      return c.json({ ok: true, data: imported[0] });
+    }
+    return c.json({ ok: true, data: { imported } });
   });
   function closeActiveRuns() {
     for (const ac of activeRuns.values()) ac.abort();