@axlsdk/studio 0.16.1 → 0.17.1

package/dist/middleware.cjs

@@ -34,9 +34,9 @@ __export(middleware_exports, {
   handleWsMessage: () => handleWsMessage
 });
 module.exports = __toCommonJS(middleware_exports);
-var import_node_path4 = require("path");
-var import_node_fs4 = require("fs");
-var import_node_url3 = require("url");
+var import_node_path3 = require("path");
+var import_node_fs2 = require("fs");
+var import_node_url2 = require("url");
 var import_node_server = require("@hono/node-server");
 var import_ws = require("ws");
 
@@ -142,8 +142,11 @@ function redactEvalItem(item) {
 }
 function redactEvalResult(result, redact) {
   if (!redact) return result;
+  const meta = result.metadata;
+  const scrubbedMetadata = meta && typeof meta.batchFailure === "string" ? { ...meta, batchFailure: REDACTED } : result.metadata;
   return {
     ...result,
+    metadata: scrubbedMetadata,
     items: result.items.map(redactEvalItem)
   };
 }
@@ -938,13 +941,18 @@ function reduceEvalTrends(acc, entry) {
   const cost = extractCost(entry.data);
   const model = extractModel(entry.data);
   const duration = extractDuration(entry.data);
+  const metadata = entry.data?.metadata;
+  const runGroupId = typeof metadata?.runGroupId === "string" ? metadata.runGroupId : void 0;
+  const batchAttempted = typeof metadata?.batchAttempted === "number" && Number.isFinite(metadata.batchAttempted) ? metadata.batchAttempted : void 0;
   const run = {
     timestamp: entry.timestamp,
     id: entry.id,
     scores,
     cost,
     ...model !== void 0 ? { model } : {},
-    ...duration !== void 0 ? { duration } : {}
+    ...duration !== void 0 ? { duration } : {},
+    ...runGroupId !== void 0 ? { runGroupId } : {},
+    ...batchAttempted !== void 0 ? { batchAttempted } : {}
   };
   const byEval = { ...acc.byEval };
   const prev = byEval[entry.eval];
@@ -1627,33 +1635,79 @@ function createEvalRoutes(connMgr, evalLoader) {
           if (runs > 1) {
             const runGroupId = (0, import_node_crypto.randomUUID)();
             const results = [];
+            let runFailure;
+            let cancelled = false;
             for (let r = 0; r < runs; r++) {
-              if (ac.signal.aborted) break;
-              const result = await runtime.runRegisteredEval(name, {
-                metadata: { runGroupId, runIndex: r },
-                signal: ac.signal,
-                captureTraces,
-                onProgress: (event) => {
-                  if (event.type === "run_done") return;
+              if (ac.signal.aborted) {
+                cancelled = true;
+                break;
+              }
+              try {
+                const result = await runtime.runRegisteredEval(name, {
+                  metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+                  signal: ac.signal,
+                  captureTraces,
+                  onProgress: (event) => {
+                    if (event.type === "run_done") return;
+                    connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                      ...event,
+                      run: r + 1,
+                      totalRuns: runs
+                    });
+                  }
+                });
+                results.push(result);
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_done",
+                  run: r + 1,
+                  totalRuns: runs
+                });
+              } catch (err) {
+                const isAbort = ac.signal.aborted || err instanceof Error && err.name === "AbortError";
+                if (isAbort) {
+                  cancelled = true;
                   connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                    ...event,
+                    type: "run_cancelled",
                     run: r + 1,
                     totalRuns: runs
                   });
+                  break;
                 }
-              });
-              results.push(result);
-              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                type: "run_done",
-                run: r + 1,
-                totalRuns: runs
-              });
+                runFailure = err instanceof Error ? err : new Error(String(err));
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_failed",
+                  run: r + 1,
+                  totalRuns: runs,
+                  message: redactErrorMessage(runFailure, redactOn)
+                });
+                break;
+              }
             }
             if (results.length > 0) {
+              const partial = results.length < runs;
+              const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
                 type: "done",
                 evalResultId: results[0].id,
-                runGroupId
+                runGroupId,
+                ...partial && {
+                  partial: true,
+                  batchCompleted: results.length,
+                  batchAttempted: runs,
+                  // `cancelled` and `batchFailure` are mutually exclusive:
+                  // the catch block sets at most one of {cancelled,
+                  // runFailure}. The client uses `cancelled` to render a
+                  // neutral "Cancelled — X of N runs completed" caption
+                  // instead of the amber "Stopped after: <message>"
+                  // failure caption.
+                  ...cancelled ? { cancelled: true } : {},
+                  ...failureMsg ? { batchFailure: failureMsg } : {}
+                }
+              });
+            } else if (runFailure) {
+              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                type: "error",
+                message: redactErrorMessage(runFailure, redactOn)
               });
             } else {
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
@@ -1691,19 +1745,38 @@ function createEvalRoutes(connMgr, evalLoader) {
         const { aggregateRuns } = await import("@axlsdk/eval");
         const runGroupId = (0, import_node_crypto.randomUUID)();
         const results = [];
+        let runFailure;
         for (let r = 0; r < runs; r++) {
-          const result2 = await runtime.runRegisteredEval(name, {
-            metadata: { runGroupId, runIndex: r },
-            captureTraces
-          });
-          results.push(result2);
+          try {
+            const result2 = await runtime.runRegisteredEval(name, {
+              metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+              captureTraces
+            });
+            results.push(result2);
+          } catch (err) {
+            runFailure = err instanceof Error ? err : new Error(String(err));
+            break;
+          }
+        }
+        if (results.length === 0) {
+          throw runFailure ?? new Error("No runs completed");
         }
-        const typedResults = results;
-        const aggregate = aggregateRuns(typedResults);
-        const first = typedResults[0];
+        const aggregate = aggregateRuns(results);
+        const first = results[0];
+        const partial = results.length < runs;
+        const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
         const result = {
           ...first,
-          _multiRun: { aggregate, allRuns: typedResults }
+          _multiRun: {
+            aggregate,
+            allRuns: results,
+            ...partial && {
+              partial: true,
+              batchCompleted: results.length,
+              batchAttempted: runs,
+              ...failureMsg ? { batchFailure: failureMsg } : {}
+            }
+          }
         };
         return c.json({
           ok: true,
@@ -1875,60 +1948,75 @@ function createEvalRoutes(connMgr, evalLoader) {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     const bad = (message) => c.json({ ok: false, error: { code: "BAD_REQUEST", message } }, 400);
-    if (!body.result || typeof body.result !== "object") {
+    if (body.result === void 0 || body.result === null) {
       return bad("result is required");
     }
-    const result = body.result;
-    if (!Array.isArray(result.items)) {
-      return bad("result.items must be an array");
-    }
-    if (typeof result.summary !== "object" || result.summary == null) {
-      return bad("result.summary must be an object");
-    }
-    if (typeof result.dataset !== "string" || !result.dataset) {
-      return bad("result.dataset must be a non-empty string (required for compare)");
-    }
-    const summary = result.summary;
-    if (typeof summary.scorers !== "object" || summary.scorers == null) {
-      return bad("result.summary.scorers must be an object");
-    }
-    const summaryScorerNames = Object.keys(summary.scorers);
-    const items = result.items;
-    const summaryScorerSet = new Set(summaryScorerNames);
-    const uncoveredAcrossItems = /* @__PURE__ */ new Set();
-    for (const item of items) {
-      const itemScores = item?.scores;
-      if (itemScores && typeof itemScores === "object") {
-        for (const name of Object.keys(itemScores)) {
-          if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+    const resultsRaw = Array.isArray(body.result) ? body.result : [body.result];
+    if (resultsRaw.length === 0) {
+      return bad("result must be a non-empty array or object");
+    }
+    const validatedResults = [];
+    for (let i = 0; i < resultsRaw.length; i++) {
+      const entry = resultsRaw[i];
+      const prefix = resultsRaw.length > 1 ? `result[${i}]` : "result";
+      if (!entry || typeof entry !== "object") {
+        return bad(`${prefix} must be an object`);
+      }
+      const r = entry;
+      if (!Array.isArray(r.items)) {
+        return bad(`${prefix}.items must be an array`);
+      }
+      if (typeof r.summary !== "object" || r.summary == null) {
+        return bad(`${prefix}.summary must be an object`);
+      }
+      if (typeof r.dataset !== "string" || !r.dataset) {
+        return bad(`${prefix}.dataset must be a non-empty string (required for compare)`);
+      }
+      const summary = r.summary;
+      if (typeof summary.scorers !== "object" || summary.scorers == null) {
+        return bad(`${prefix}.summary.scorers must be an object`);
+      }
+      const summaryScorerNames = Object.keys(summary.scorers);
+      const items = r.items;
+      const summaryScorerSet = new Set(summaryScorerNames);
+      const uncoveredAcrossItems = /* @__PURE__ */ new Set();
+      for (const item of items) {
+        const itemScores = item?.scores;
+        if (itemScores && typeof itemScores === "object") {
+          for (const name of Object.keys(itemScores)) {
+            if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+          }
         }
       }
-    }
-    if (uncoveredAcrossItems.size > 0) {
-      return bad(
-        `item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
-      );
+      if (uncoveredAcrossItems.size > 0) {
+        return bad(
+          `${prefix} item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
+        );
+      }
+      validatedResults.push(r);
     }
     const trim = (v) => typeof v === "string" && v.trim() !== "" ? v.trim() : void 0;
-    const metadataObj = typeof result.metadata === "object" && result.metadata != null ? result.metadata : {};
+    const firstResult = validatedResults[0];
+    const metadataObj = typeof firstResult.metadata === "object" && firstResult.metadata != null ? firstResult.metadata : {};
     const workflowsFromMeta = Array.isArray(metadataObj.workflows) ? metadataObj.workflows : [];
     const primaryWorkflow = workflowsFromMeta.find((w) => typeof w === "string");
-    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? // Legacy fallback: pre-0.14 CLI artifacts had workflow at the top level.
-    trim(result.workflow) ?? "imported";
-    const id = (0, import_node_crypto.randomUUID)();
+    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? trim(firstResult.workflow) ?? "imported";
     const timestamp = Date.now();
-    const imported = {
-      ...result,
-      id,
-      metadata: typeof result.metadata === "object" && result.metadata != null ? result.metadata : {}
-    };
-    await runtime.saveEvalResult({
-      id,
-      eval: evalName,
-      timestamp,
-      data: imported
-    });
-    return c.json({ ok: true, data: { id, eval: evalName, timestamp } });
+    const imported = [];
+    for (const r of validatedResults) {
+      const id = (0, import_node_crypto.randomUUID)();
+      const entry = {
+        ...r,
+        id,
+        metadata: typeof r.metadata === "object" && r.metadata != null ? r.metadata : {}
+      };
+      await runtime.saveEvalResult({ id, eval: evalName, timestamp, data: entry });
+      imported.push({ id, eval: evalName, timestamp });
+    }
+    if (imported.length === 1) {
+      return c.json({ ok: true, data: imported[0] });
+    }
+    return c.json({ ok: true, data: { imported } });
   });
   function closeActiveRuns() {
     for (const ac of activeRuns.values()) ac.abort();
@@ -2245,41 +2333,11 @@ function createServer(options) {
 }
 
 // src/eval-loader.ts
-var import_node_path3 = require("path");
-var import_node_fs3 = require("fs");
-var import_node_url2 = require("url");
-
-// src/cli-utils.ts
 var import_node_path2 = require("path");
-var import_node_fs2 = require("fs");
 var import_node_url = require("url");
-function needsTsxLoader(configPath) {
-  return /\.[mc]?tsx?$/.test(configPath);
-}
-var tsImportFn;
-async function importModule(filePath, parentURL2) {
-  if (needsTsxLoader(filePath)) {
-    if (tsImportFn === void 0) {
-      try {
-        const mod = await import("tsx/esm/api");
-        tsImportFn = mod.tsImport ?? null;
-      } catch {
-        tsImportFn = null;
-        console.warn(
-          "[axl-studio] Warning: tsx is not installed. TypeScript config files require tsx as a dependency.\n  Install it with: npm install -D tsx"
-        );
-      }
-    }
-    if (tsImportFn) {
-      return await tsImportFn((0, import_node_url.pathToFileURL)(filePath).href, parentURL2);
-    }
-  }
-  return await import((0, import_node_url.pathToFileURL)(filePath).href);
-}
-
-// src/eval-loader.ts
+var import_axl6 = require("@axlsdk/axl");
 var import_meta = {};
-var parentURL = import_meta.url ?? (0, import_node_url2.pathToFileURL)(typeof __filename !== "undefined" ? __filename : __dirname).href;
+var parentURL = import_meta.url ?? (0, import_node_url.pathToFileURL)(typeof __filename !== "undefined" ? __filename : __dirname).href;
 function createEvalLoader(config, runtime, cwd) {
   let loadPromise;
   const { patterns, conditions } = normalizeConfig(config);
@@ -2296,7 +2354,7 @@ function createEvalLoader(config, runtime, cwd) {
 }
 async function loadEvalFiles(patterns, conditions, cwd, runtime) {
   if (conditions.length > 0) {
-    await registerConditions(conditions);
+    await (0, import_axl6.registerConditions)(conditions);
   }
   const files = resolvePatterns(patterns, cwd);
   if (files.length === 0) {
@@ -2305,11 +2363,30 @@ async function loadEvalFiles(patterns, conditions, cwd, runtime) {
   }
   for (const file of files) {
     try {
-      const mod = await importModule(file, parentURL);
-      const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
+      const mod = await (0, import_axl6.importModule)(file, parentURL);
+      const evalConfig = (0, import_axl6.pickDefault)(mod);
+      if (!evalConfig || typeof evalConfig !== "object") {
+        console.warn(
+          `[axl-studio] Skipping ${file}: default export is ${evalConfig === null ? "null" : typeof evalConfig}, expected an eval config object.`
+        );
+        continue;
+      }
+      if (Array.isArray(evalConfig.scorers) && evalConfig.scorers.length === 0) {
+        console.warn(
+          `[axl-studio] Skipping ${file}: scorers array is empty \u2014 at least one scorer is required.`
+        );
+        continue;
+      }
       if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        const missing = [
+          !evalConfig.workflow && "workflow",
+          !evalConfig.dataset && "dataset",
+          !evalConfig.scorers && "scorers"
+        ].filter(Boolean).join(", ");
+        const keys = Object.keys(evalConfig).slice(0, 10);
+        const got = keys.length > 0 ? ` Got: { ${keys.join(", ")} }.` : "";
         console.warn(
-          `[axl-studio] Skipping ${file}: not a valid eval config (missing workflow, dataset, or scorers)`
+          `[axl-studio] Skipping ${file}: not a valid eval config (missing ${missing}).${got}`
         );
         continue;
       }
@@ -2319,7 +2396,16 @@ async function loadEvalFiles(patterns, conditions, cwd, runtime) {
           `[axl-studio] Eval name "${name}" from ${file} collides with an already-registered eval \u2014 overwriting`
         );
       }
-      runtime.registerEval(name, evalConfig, mod.executeWorkflow);
+      const exported = (0, import_axl6.pickExport)(mod, "executeWorkflow");
+      let customExecute;
+      if (typeof exported === "function") {
+        customExecute = exported;
+      } else if (exported !== void 0) {
+        console.warn(
+          `[axl-studio] ${file} exports executeWorkflow but it is ${typeof exported}, not a function \u2014 ignoring and falling back to runtime.execute()`
+        );
+      }
+      runtime.registerEval(name, evalConfig, customExecute);
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       console.warn(`[axl-studio] Failed to load eval ${file}: ${msg}`);
@@ -2337,10 +2423,10 @@ function normalizeConfig(config) {
   return { patterns: files, conditions: config.conditions ?? [] };
 }
 function deriveEvalName(filePath, cwd) {
-  const rel = (0, import_node_path3.relative)(cwd, filePath);
+  const rel = (0, import_node_path2.relative)(cwd, filePath);
   const normalized = rel.replace(/\\/g, "/");
   if (normalized.startsWith("../")) {
-    const base = (0, import_node_path3.basename)(filePath);
+    const base = (0, import_node_path2.basename)(filePath);
     const stripped = base.replace(/\.eval\.[mc]?[jt]sx?$/, "");
     return stripped !== base ? stripped : base.replace(/\.[mc]?[jt]sx?$/, "") || base;
   }
@@ -2353,7 +2439,7 @@ function resolvePatterns(patterns, cwd) {
   const files = [];
   const seen = /* @__PURE__ */ new Set();
   for (const pattern of patterns) {
-    const resolved = pattern.includes("*") ? expandGlob(pattern, cwd) : [(0, import_node_path3.resolve)(cwd, pattern)];
+    const resolved = pattern.includes("*") ? (0, import_axl6.expandGlob)(pattern, cwd) : [(0, import_node_path2.resolve)(cwd, pattern)];
     for (const file of resolved) {
       if (!seen.has(file)) {
         seen.add(file);
@@ -2363,61 +2449,6 @@ function resolvePatterns(patterns, cwd) {
   }
   return files;
 }
-function expandGlob(pattern, cwd) {
-  if (pattern.includes("**/")) {
-    const sepIdx = pattern.indexOf("**/");
-    const baseDir = (0, import_node_path3.resolve)(cwd, pattern.slice(0, sepIdx) || ".");
-    const fileGlob2 = pattern.slice(sepIdx + 3) || "*";
-    return findFiles(baseDir, fileGlob2, true);
-  }
-  const dir = (0, import_node_path3.resolve)(cwd, (0, import_node_path3.dirname)(pattern));
-  const fileGlob = (0, import_node_path3.basename)(pattern);
-  return findFiles(dir, fileGlob, false);
-}
-var MAX_DEPTH = 20;
-function findFiles(dir, fileGlob, recursive, depth = 0) {
-  if (depth > MAX_DEPTH) return [];
-  const matcher = globToRegex(fileGlob);
-  const results = [];
-  try {
-    const entries = (0, import_node_fs3.readdirSync)(dir);
-    for (const entry of entries) {
-      const full = (0, import_node_path3.resolve)(dir, entry);
-      try {
-        const stat = (0, import_node_fs3.statSync)(full);
-        if (stat.isFile() && matcher.test(entry)) {
-          results.push(full);
-        } else if (stat.isDirectory() && recursive) {
-          results.push(...findFiles(full, fileGlob, true, depth + 1));
-        }
-      } catch {
-      }
-    }
-  } catch {
-  }
-  return results;
-}
-function globToRegex(glob) {
-  const escaped = glob.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  return new RegExp(`^${escaped}$`);
-}
-async function registerConditions(conditions) {
-  try {
-    const nodeModule = await import("module");
-    const hookCode = [
-      `const extra = ${JSON.stringify(conditions)};`,
-      `export async function resolve(specifier, context, nextResolve) {`,
-      `  return nextResolve(specifier, {`,
-      `    ...context,`,
-      `    conditions: [...new Set([...context.conditions, ...extra])],`,
-      `  });`,
-      `}`
-    ].join("\n");
-    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
-  } catch {
-    console.warn("[axl-studio] Warning: import conditions require Node.js 20.6+");
-  }
-}
 
 // src/middleware.ts
 var import_meta2 = {};
@@ -2432,9 +2463,9 @@ function createStudioMiddleware(options) {
   const basePath = normalizeBasePath(options.basePath);
   const staticRoot = serveClient ? resolveClientDist() : void 0;
   if (serveClient && !staticRoot) {
-    const dir = import_meta2.dirname ?? (typeof __dirname !== "undefined" ? __dirname : (0, import_node_path4.dirname)((0, import_node_url3.fileURLToPath)(import_meta2.url)));
+    const dir = import_meta2.dirname ?? (typeof __dirname !== "undefined" ? __dirname : (0, import_node_path3.dirname)((0, import_node_url2.fileURLToPath)(import_meta2.url)));
     console.warn(
-      `[axl-studio] serveClient is true but no pre-built client found at ${(0, import_node_path4.resolve)(dir, "client")}. Studio UI will not be available. Set serveClient: false to suppress this warning.`
+      `[axl-studio] serveClient is true but no pre-built client found at ${(0, import_node_path3.resolve)(dir, "client")}. Studio UI will not be available. Set serveClient: false to suppress this warning.`
     );
   }
   const evalLoader = options.evals ? createEvalLoader(options.evals, runtime) : void 0;
@@ -2607,9 +2638,9 @@ function normalizeBasePath(raw) {
   return normalized;
 }
 function resolveClientDist() {
-  const dir = import_meta2.dirname ?? (typeof __dirname !== "undefined" ? __dirname : (0, import_node_path4.dirname)((0, import_node_url3.fileURLToPath)(import_meta2.url)));
-  const candidate = (0, import_node_path4.resolve)(dir, "client");
-  return (0, import_node_fs4.existsSync)((0, import_node_path4.resolve)(candidate, "index.html")) ? candidate : void 0;
+  const dir = import_meta2.dirname ?? (typeof __dirname !== "undefined" ? __dirname : (0, import_node_path3.dirname)((0, import_node_url2.fileURLToPath)(import_meta2.url)));
+  const candidate = (0, import_node_path3.resolve)(dir, "client");
+  return (0, import_node_fs2.existsSync)((0, import_node_path3.resolve)(candidate, "index.html")) ? candidate : void 0;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {