@axlsdk/eval 0.13.8 → 0.15.0

package/dist/cli.cjs

@@ -311,7 +311,36 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, runtime) {
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
@@ -331,6 +360,17 @@ async function runEval(config, executeWorkflow, runtime) {
   let totalCost = 0;
   let budgetExceeded = false;
   async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = {
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Cancelled",
+        scores: {}
+      };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
     if (budgetExceeded) {
       evalItems[itemIndex] = {
         input: item.input,
@@ -339,6 +379,7 @@ async function runEval(config, executeWorkflow, runtime) {
         error: "Budget exceeded",
         scores: {}
       };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     const evalItem = {
@@ -349,18 +390,43 @@ async function runEval(config, executeWorkflow, runtime) {
     };
     const itemStart = Date.now();
     try {
-      const result = await executeWorkflow(item.input, runtime);
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.output = result.output;
-      evalItem.cost = result.cost;
-      if (result.metadata) evalItem.metadata = result.metadata;
-      if (result.cost != null) {
-        totalCost += result.cost;
+      if (options?.captureTraces) {
+        const tracked = await runtime.trackExecution(
+          async () => executeWorkflow(item.input, runtime),
+          { captureTraces: true }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = tracked.result.output;
+        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
+        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        if (tracked.traces && tracked.traces.length > 0) {
+          evalItem.traces = tracked.traces;
+        }
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
+      } else {
+        const result = await executeWorkflow(item.input, runtime);
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = result.output;
+        evalItem.cost = extractUserCost(result);
+        const meta = extractUserMetadata(result);
+        if (meta) evalItem.metadata = meta;
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
       }
     } catch (err) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.error = err instanceof Error ? err.message : String(err);
+      if (options?.captureTraces) {
+        const captured = err.axlCapturedTraces;
+        if (Array.isArray(captured) && captured.length > 0) {
+          evalItem.traces = captured;
+        }
+      }
       evalItems[itemIndex] = evalItem;
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     if (budgetLimit != null && totalCost > budgetLimit) {
@@ -369,6 +435,7 @@ async function runEval(config, executeWorkflow, runtime) {
     evalItem.scoreDetails = {};
     let itemScorerCost = 0;
     for (const scorer of config.scorers) {
+      if (options?.signal?.aborted) break;
       const scorerStart = Date.now();
       try {
         const raw = await scorer.score(
@@ -424,6 +491,7 @@ async function runEval(config, executeWorkflow, runtime) {
     }
     evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
     evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
   let index = 0;
   async function runNext() {
@@ -470,14 +538,42 @@ async function runEval(config, executeWorkflow, runtime) {
     modelsMeta.models = sorted.map(([m]) => m);
     modelsMeta.modelCounts = Object.fromEntries(sorted);
   }
+  const totalWorkflowCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.workflowCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [w, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
+      }
+    } else {
+      const itemWorkflows = item.metadata?.workflows;
+      if (Array.isArray(itemWorkflows)) {
+        for (const w of itemWorkflows) {
+          if (typeof w === "string")
+            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const workflowsMeta = {};
+  if (totalWorkflowCalls.size > 0) {
+    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
+    workflowsMeta.workflows = sorted.map(([w]) => w);
+    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
+  } else if (config.workflow) {
+    workflowsMeta.workflows = [config.workflow];
+    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
+  }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
   return {
     id,
-    workflow: config.workflow,
     dataset: config.dataset.name,
     metadata: {
       ...config.metadata,
       scorerTypes,
-      ...modelsMeta
+      ...modelsMeta,
+      ...workflowsMeta
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
@@ -520,6 +616,7 @@ async function rescore(result, scorers, runtime, options) {
       annotations: original.annotations,
       output: original.output,
       metadata: original.metadata,
+      traces: original.traces,
       scores: {},
       scoreDetails: {}
     };
@@ -601,11 +698,23 @@ async function rescore(result, scorers, runtime, options) {
   }
   return {
     id: (0, import_node_crypto2.randomUUID)(),
-    workflow: result.workflow,
     dataset: result.dataset,
     metadata: (() => {
       const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
-      return { ...rest, rescored: true, originalId: result.id, scorerTypes };
+      const merged = {
+        ...rest,
+        rescored: true,
+        originalId: result.id,
+        scorerTypes
+      };
+      if (!Array.isArray(merged.workflows)) {
+        const legacyWorkflow = result.workflow;
+        if (typeof legacyWorkflow === "string" && legacyWorkflow) {
+          merged.workflows = [legacyWorkflow];
+          merged.workflowCounts = { [legacyWorkflow]: result.items.length };
+        }
+      }
+      return merged;
     })(),
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
@@ -625,7 +734,20 @@ function std(values, mean) {
 function aggregateRuns(runs) {
   if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
   const runGroupId = runs[0].metadata.runGroupId ?? (0, import_node_crypto3.randomUUID)();
-  const { workflow, dataset } = runs[0];
+  const { dataset } = runs[0];
+  const seenWorkflows = /* @__PURE__ */ new Set();
+  const workflows = [];
+  for (const run of runs) {
+    const list = run.metadata.workflows;
+    if (Array.isArray(list)) {
+      for (const w of list) {
+        if (typeof w === "string" && !seenWorkflows.has(w)) {
+          seenWorkflows.add(w);
+          workflows.push(w);
+        }
+      }
+    }
+  }
   const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
   const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
   const scorerNames = Object.keys(runs[0].summary.scorers);
@@ -654,7 +776,7 @@ function aggregateRuns(runs) {
   return {
     runGroupId,
     runCount: runs.length,
-    workflow,
+    workflows,
     dataset,
     totalCost: round3(totalCost),
     totalDuration: round3(totalDuration),
@@ -733,7 +855,8 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--conditions",
   "--fail-on-regression",
   "--threshold",
-  "--runs"
+  "--runs",
+  "--capture-traces"
 ]);
 async function main() {
   const args = process.argv.slice(2);
@@ -745,6 +868,10 @@ Usage:
   axl-eval <path> --output <file>         Save results to JSON
   axl-eval <path> --config <file>         Use config file for runtime
   axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval <path> --capture-traces        Populate EvalItem.traces on every item
+                                          (success + failure). Adds memory
+                                          overhead proportional to dataset size
+                                          x turns x agents; off by default.
   axl-eval rescore <results> <eval-file>  Re-run scorers on saved outputs
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
@@ -911,12 +1038,18 @@ function collectEvalFiles(p) {
     return [resolved];
   }
 }
+function formatWorkflows(workflows) {
+  if (!Array.isArray(workflows) || workflows.length === 0) return "(unknown)";
+  return workflows.filter((w) => typeof w === "string").join(", ");
+}
 function formatTable(result) {
   const lines = [];
   const scorerNames = Object.keys(result.summary.scorers);
   const maxNameLen = Math.max(...scorerNames.map((n) => n.length), "Scorer".length);
   const colWidth = 8;
-  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);
+  lines.push(
+    `Eval: ${formatWorkflows(result.metadata.workflows)} x ${result.dataset} (${result.summary.count} items)`
+  );
   lines.push(
     `  ${"Scorer".padEnd(maxNameLen)}  ${"Mean".padStart(colWidth)}  ${"Min".padStart(colWidth)}  ${"Max".padStart(colWidth)}  ${"p50".padStart(colWidth)}  ${"p95".padStart(colWidth)}`
   );
@@ -968,7 +1101,9 @@ function formatMultiRunTable(summary) {
   const scorerNames = Object.keys(summary.scorers);
   const maxNameLen = Math.max(...scorerNames.map((n) => n.length), "Scorer".length);
   const colWidth = 16;
-  lines.push(`Eval: ${summary.workflow} x ${summary.dataset} \u2014 ${summary.runCount} runs`);
+  lines.push(
+    `Eval: ${formatWorkflows(summary.workflows)} x ${summary.dataset} \u2014 ${summary.runCount} runs`
+  );
   lines.push(
     `  ${"Scorer".padEnd(maxNameLen)}  ${"Mean \xB1 Std".padStart(colWidth)}  ${"Min".padStart(8)}  ${"Max".padStart(8)}`
   );
@@ -1057,6 +1192,7 @@ function parseEvalArgs(args) {
   let configArg;
   let conditions = [];
   let runs = 1;
+  let captureTraces = false;
   const paths = [];
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -1071,6 +1207,8 @@ function parseEvalArgs(args) {
       else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
       else
         conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
+    } else if (arg === "--capture-traces") {
+      captureTraces = true;
     } else if (arg.startsWith("--")) {
       if (!KNOWN_FLAGS.has(arg)) {
         console.error(`Unknown flag: ${arg}`);
@@ -1080,10 +1218,10 @@ function parseEvalArgs(args) {
       paths.push(arg);
     }
   }
-  return { outputPath, configArg, conditions, runs, paths };
+  return { outputPath, configArg, conditions, runs, captureTraces, paths };
 }
 async function runEvalCommand(args) {
-  const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);
+  const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -1138,13 +1276,14 @@ async function runEvalCommand(args) {
           );
           executeWorkflow = async (input) => ({ output: input });
         }
+        const runOptions = captureTraces ? { captureTraces: true } : void 0;
         if (runs > 1) {
           const { randomUUID: randomUUID4 } = await import("crypto");
           const runGroupId = randomUUID4();
           const runResults = [];
           for (let r = 0; r < runs; r++) {
             console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
-            const result = await runEval(evalConfig, executeWorkflow, runtime);
+            const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
             result.metadata.runGroupId = runGroupId;
             result.metadata.runIndex = r;
             runResults.push(result);
@@ -1153,7 +1292,7 @@ async function runEvalCommand(args) {
           const summary = aggregateRuns(runResults);
           console.log("\n" + formatMultiRunTable(summary) + "\n");
         } else {
-          const result = await runEval(evalConfig, executeWorkflow, runtime);
+          const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
         }