@axlsdk/studio 0.16.1 → 0.17.0

package/dist/cli.cjs

@@ -24,8 +24,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/cli.ts
-var import_node_path3 = require("path");
-var import_node_fs3 = require("fs");
+var import_node_path2 = require("path");
+var import_node_fs2 = require("fs");
 var import_node_server = require("@hono/node-server");
 var import_node_ws = require("@hono/node-ws");
 
@@ -131,8 +131,11 @@ function redactEvalItem(item) {
 }
 function redactEvalResult(result, redact) {
   if (!redact) return result;
+  const meta = result.metadata;
+  const scrubbedMetadata = meta && typeof meta.batchFailure === "string" ? { ...meta, batchFailure: REDACTED } : result.metadata;
   return {
     ...result,
+    metadata: scrubbedMetadata,
     items: result.items.map(redactEvalItem)
   };
 }
@@ -927,13 +930,18 @@ function reduceEvalTrends(acc, entry) {
   const cost = extractCost(entry.data);
   const model = extractModel(entry.data);
   const duration = extractDuration(entry.data);
+  const metadata = entry.data?.metadata;
+  const runGroupId = typeof metadata?.runGroupId === "string" ? metadata.runGroupId : void 0;
+  const batchAttempted = typeof metadata?.batchAttempted === "number" && Number.isFinite(metadata.batchAttempted) ? metadata.batchAttempted : void 0;
   const run = {
     timestamp: entry.timestamp,
     id: entry.id,
     scores,
     cost,
     ...model !== void 0 ? { model } : {},
-    ...duration !== void 0 ? { duration } : {}
+    ...duration !== void 0 ? { duration } : {},
+    ...runGroupId !== void 0 ? { runGroupId } : {},
+    ...batchAttempted !== void 0 ? { batchAttempted } : {}
   };
   const byEval = { ...acc.byEval };
   const prev = byEval[entry.eval];
@@ -1616,33 +1624,79 @@ function createEvalRoutes(connMgr, evalLoader) {
           if (runs > 1) {
             const runGroupId = (0, import_node_crypto.randomUUID)();
             const results = [];
+            let runFailure;
+            let cancelled = false;
             for (let r = 0; r < runs; r++) {
-              if (ac.signal.aborted) break;
-              const result = await runtime.runRegisteredEval(name, {
-                metadata: { runGroupId, runIndex: r },
-                signal: ac.signal,
-                captureTraces,
-                onProgress: (event) => {
-                  if (event.type === "run_done") return;
+              if (ac.signal.aborted) {
+                cancelled = true;
+                break;
+              }
+              try {
+                const result = await runtime.runRegisteredEval(name, {
+                  metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+                  signal: ac.signal,
+                  captureTraces,
+                  onProgress: (event) => {
+                    if (event.type === "run_done") return;
+                    connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                      ...event,
+                      run: r + 1,
+                      totalRuns: runs
+                    });
+                  }
+                });
+                results.push(result);
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_done",
+                  run: r + 1,
+                  totalRuns: runs
+                });
+              } catch (err) {
+                const isAbort = ac.signal.aborted || err instanceof Error && err.name === "AbortError";
+                if (isAbort) {
+                  cancelled = true;
                   connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                    ...event,
+                    type: "run_cancelled",
                     run: r + 1,
                     totalRuns: runs
                   });
+                  break;
                 }
-              });
-              results.push(result);
-              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
-                type: "run_done",
-                run: r + 1,
-                totalRuns: runs
-              });
+                runFailure = err instanceof Error ? err : new Error(String(err));
+                connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                  type: "run_failed",
+                  run: r + 1,
+                  totalRuns: runs,
+                  message: redactErrorMessage(runFailure, redactOn)
+                });
+                break;
+              }
             }
             if (results.length > 0) {
+              const partial = results.length < runs;
+              const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
                 type: "done",
                 evalResultId: results[0].id,
-                runGroupId
+                runGroupId,
+                ...partial && {
+                  partial: true,
+                  batchCompleted: results.length,
+                  batchAttempted: runs,
+                  // `cancelled` and `batchFailure` are mutually exclusive:
+                  // the catch block sets at most one of {cancelled,
+                  // runFailure}. The client uses `cancelled` to render a
+                  // neutral "Cancelled — X of N runs completed" caption
+                  // instead of the amber "Stopped after: <message>"
+                  // failure caption.
+                  ...cancelled ? { cancelled: true } : {},
+                  ...failureMsg ? { batchFailure: failureMsg } : {}
+                }
+              });
+            } else if (runFailure) {
+              connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
+                type: "error",
+                message: redactErrorMessage(runFailure, redactOn)
               });
             } else {
               connMgr.broadcastWithWildcard(`eval:${evalRunId}`, {
@@ -1680,19 +1734,38 @@ function createEvalRoutes(connMgr, evalLoader) {
         const { aggregateRuns } = await import("@axlsdk/eval");
         const runGroupId = (0, import_node_crypto.randomUUID)();
         const results = [];
+        let runFailure;
         for (let r = 0; r < runs; r++) {
-          const result2 = await runtime.runRegisteredEval(name, {
-            metadata: { runGroupId, runIndex: r },
-            captureTraces
-          });
-          results.push(result2);
+          try {
+            const result2 = await runtime.runRegisteredEval(name, {
+              metadata: { runGroupId, runIndex: r, batchAttempted: runs },
+              captureTraces
+            });
+            results.push(result2);
+          } catch (err) {
+            runFailure = err instanceof Error ? err : new Error(String(err));
+            break;
+          }
+        }
+        if (results.length === 0) {
+          throw runFailure ?? new Error("No runs completed");
         }
-        const typedResults = results;
-        const aggregate = aggregateRuns(typedResults);
-        const first = typedResults[0];
+        const aggregate = aggregateRuns(results);
+        const first = results[0];
+        const partial = results.length < runs;
+        const failureMsg = runFailure ? redactErrorMessage(runFailure, redactOn) || String(runFailure) || void 0 : void 0;
         const result = {
           ...first,
-          _multiRun: { aggregate, allRuns: typedResults }
+          _multiRun: {
+            aggregate,
+            allRuns: results,
+            ...partial && {
+              partial: true,
+              batchCompleted: results.length,
+              batchAttempted: runs,
+              ...failureMsg ? { batchFailure: failureMsg } : {}
+            }
+          }
         };
         return c.json({
           ok: true,
@@ -1864,60 +1937,75 @@ function createEvalRoutes(connMgr, evalLoader) {
     const runtime = c.get("runtime");
     const body = await c.req.json();
     const bad = (message) => c.json({ ok: false, error: { code: "BAD_REQUEST", message } }, 400);
-    if (!body.result || typeof body.result !== "object") {
+    if (body.result === void 0 || body.result === null) {
       return bad("result is required");
     }
-    const result = body.result;
-    if (!Array.isArray(result.items)) {
-      return bad("result.items must be an array");
-    }
-    if (typeof result.summary !== "object" || result.summary == null) {
-      return bad("result.summary must be an object");
-    }
-    if (typeof result.dataset !== "string" || !result.dataset) {
-      return bad("result.dataset must be a non-empty string (required for compare)");
-    }
-    const summary = result.summary;
-    if (typeof summary.scorers !== "object" || summary.scorers == null) {
-      return bad("result.summary.scorers must be an object");
-    }
-    const summaryScorerNames = Object.keys(summary.scorers);
-    const items = result.items;
-    const summaryScorerSet = new Set(summaryScorerNames);
-    const uncoveredAcrossItems = /* @__PURE__ */ new Set();
-    for (const item of items) {
-      const itemScores = item?.scores;
-      if (itemScores && typeof itemScores === "object") {
-        for (const name of Object.keys(itemScores)) {
-          if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+    const resultsRaw = Array.isArray(body.result) ? body.result : [body.result];
+    if (resultsRaw.length === 0) {
+      return bad("result must be a non-empty array or object");
+    }
+    const validatedResults = [];
+    for (let i = 0; i < resultsRaw.length; i++) {
+      const entry = resultsRaw[i];
+      const prefix = resultsRaw.length > 1 ? `result[${i}]` : "result";
+      if (!entry || typeof entry !== "object") {
+        return bad(`${prefix} must be an object`);
+      }
+      const r = entry;
+      if (!Array.isArray(r.items)) {
+        return bad(`${prefix}.items must be an array`);
+      }
+      if (typeof r.summary !== "object" || r.summary == null) {
+        return bad(`${prefix}.summary must be an object`);
+      }
+      if (typeof r.dataset !== "string" || !r.dataset) {
+        return bad(`${prefix}.dataset must be a non-empty string (required for compare)`);
+      }
+      const summary = r.summary;
+      if (typeof summary.scorers !== "object" || summary.scorers == null) {
+        return bad(`${prefix}.summary.scorers must be an object`);
+      }
+      const summaryScorerNames = Object.keys(summary.scorers);
+      const items = r.items;
+      const summaryScorerSet = new Set(summaryScorerNames);
+      const uncoveredAcrossItems = /* @__PURE__ */ new Set();
+      for (const item of items) {
+        const itemScores = item?.scores;
+        if (itemScores && typeof itemScores === "object") {
+          for (const name of Object.keys(itemScores)) {
+            if (!summaryScorerSet.has(name)) uncoveredAcrossItems.add(name);
+          }
         }
       }
-    }
-    if (uncoveredAcrossItems.size > 0) {
-      return bad(
-        `item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
-      );
+      if (uncoveredAcrossItems.size > 0) {
+        return bad(
+          `${prefix} item scores reference scorer(s) not in summary.scorers: ${[...uncoveredAcrossItems].join(", ")}`
+        );
+      }
+      validatedResults.push(r);
     }
     const trim = (v) => typeof v === "string" && v.trim() !== "" ? v.trim() : void 0;
-    const metadataObj = typeof result.metadata === "object" && result.metadata != null ? result.metadata : {};
+    const firstResult = validatedResults[0];
+    const metadataObj = typeof firstResult.metadata === "object" && firstResult.metadata != null ? firstResult.metadata : {};
     const workflowsFromMeta = Array.isArray(metadataObj.workflows) ? metadataObj.workflows : [];
     const primaryWorkflow = workflowsFromMeta.find((w) => typeof w === "string");
-    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? // Legacy fallback: pre-0.14 CLI artifacts had workflow at the top level.
-    trim(result.workflow) ?? "imported";
-    const id = (0, import_node_crypto.randomUUID)();
+    const evalName = trim(body.eval) ?? trim(primaryWorkflow) ?? trim(firstResult.workflow) ?? "imported";
     const timestamp = Date.now();
-    const imported = {
-      ...result,
-      id,
-      metadata: typeof result.metadata === "object" && result.metadata != null ? result.metadata : {}
-    };
-    await runtime.saveEvalResult({
-      id,
-      eval: evalName,
-      timestamp,
-      data: imported
-    });
-    return c.json({ ok: true, data: { id, eval: evalName, timestamp } });
+    const imported = [];
+    for (const r of validatedResults) {
+      const id = (0, import_node_crypto.randomUUID)();
+      const entry = {
+        ...r,
+        id,
+        metadata: typeof r.metadata === "object" && r.metadata != null ? r.metadata : {}
+      };
+      await runtime.saveEvalResult({ id, eval: evalName, timestamp, data: entry });
+      imported.push({ id, eval: evalName, timestamp });
+    }
+    if (imported.length === 1) {
+      return c.json({ ok: true, data: imported[0] });
+    }
+    return c.json({ ok: true, data: { imported } });
   });
   function closeActiveRuns() {
     for (const ac of activeRuns.values()) ac.abort();
@@ -2234,28 +2322,10 @@ function createServer(options) {
 }
 
 // src/resolve-runtime.ts
-function resolveRuntime(mod) {
-  const def = mod.default;
-  return def?.default ?? def ?? mod.runtime;
-}
+var import_axl6 = require("@axlsdk/axl");
 
 // src/cli-utils.ts
-var import_node_path2 = require("path");
-var import_node_fs2 = require("fs");
-var import_node_url = require("url");
-var CONFIG_CANDIDATES = [
-  "axl.config.mts",
-  "axl.config.ts",
-  "axl.config.mjs",
-  "axl.config.js"
-];
-function findConfig(cwd) {
-  for (const name of CONFIG_CANDIDATES) {
-    const p = (0, import_node_path2.resolve)(cwd, name);
-    if ((0, import_node_fs2.existsSync)(p)) return p;
-  }
-  return void 0;
-}
+var import_axl7 = require("@axlsdk/axl");
 function parseArgs(argv) {
   let port = 4400;
   let config;
@@ -2288,29 +2358,6 @@ function parseArgs(argv) {
   }
   return result;
 }
-function needsTsxLoader(configPath) {
-  return /\.[mc]?tsx?$/.test(configPath);
-}
-var tsImportFn;
-async function importModule(filePath, parentURL) {
-  if (needsTsxLoader(filePath)) {
-    if (tsImportFn === void 0) {
-      try {
-        const mod = await import("tsx/esm/api");
-        tsImportFn = mod.tsImport ?? null;
-      } catch {
-        tsImportFn = null;
-        console.warn(
-          "[axl-studio] Warning: tsx is not installed. TypeScript config files require tsx as a dependency.\n  Install it with: npm install -D tsx"
-        );
-      }
-    }
-    if (tsImportFn) {
-      return await tsImportFn((0, import_node_url.pathToFileURL)(filePath).href, parentURL);
-    }
-  }
-  return await import((0, import_node_url.pathToFileURL)(filePath).href);
-}
 
 // src/cli.ts
 var import_meta = {};
@@ -2332,7 +2379,7 @@ Options:
   -h, --help               Show this help message
 
 Config auto-detection order:
-  ${CONFIG_CANDIDATES.join(" \u2192 ")}
+  ${import_axl7.CONFIG_CANDIDATES.join(" \u2192 ")}
 
 Tip: Use .mts for configs with top-level await or in projects without "type": "module".
 `);
@@ -2344,15 +2391,15 @@ Tip: Use .mts for configs with top-level await or in projects without "type": "m
   }
   let configPath;
   if (args.config) {
-    configPath = (0, import_node_path3.resolve)(process.cwd(), args.config);
-    if (!(0, import_node_fs3.existsSync)(configPath)) {
+    configPath = (0, import_node_path2.resolve)(process.cwd(), args.config);
+    if (!(0, import_node_fs2.existsSync)(configPath)) {
       console.error(`Config file not found: ${configPath}`);
       process.exit(1);
     }
   } else {
-    const found = findConfig(process.cwd());
+    const found = (0, import_axl7.findConfig)(process.cwd());
     if (!found) {
-      console.error(`No config file found. Searched for: ${CONFIG_CANDIDATES.join(", ")}`);
+      console.error(`No config file found. Searched for: ${import_axl7.CONFIG_CANDIDATES.join(", ")}`);
       console.error(`Create an axl.config.mts that exports a default AxlRuntime instance.`);
       process.exit(1);
     }
@@ -2377,10 +2424,10 @@ Tip: Use .mts for configs with top-level await or in projects without "type": "m
   }
   console.log(`[axl-studio] Loading config from ${configPath}`);
   let runtime;
-  const ext = (0, import_node_path3.extname)(configPath);
+  const ext = (0, import_node_path2.extname)(configPath);
   try {
-    const mod = await importModule(configPath, import_meta.url);
-    runtime = resolveRuntime(mod);
+    const mod = await (0, import_axl7.importModule)(configPath, import_meta.url);
+    runtime = (0, import_axl6.resolveRuntime)(mod);
     if (!runtime || typeof runtime.execute !== "function") {
       console.error(`Config must export a default AxlRuntime instance.`);
       if (runtime) {
@@ -2412,8 +2459,8 @@ Tip: Use .mts for configs with top-level await or in projects without "type": "m
     console.error(`Failed to load config:`, err);
     process.exit(1);
   }
-  const staticRoot = (0, import_node_path3.resolve)(import_meta.dirname ?? __dirname, "client");
-  const hasStaticAssets = (0, import_node_fs3.existsSync)((0, import_node_path3.resolve)(staticRoot, "index.html"));
+  const staticRoot = (0, import_node_path2.resolve)(import_meta.dirname ?? __dirname, "client");
+  const hasStaticAssets = (0, import_node_fs2.existsSync)((0, import_node_path2.resolve)(staticRoot, "index.html"));
   const { app: app6, createWsHandlers: createWsHandlers2 } = createServer({
     runtime,
     staticRoot: hasStaticAssets ? staticRoot : void 0,