@axlsdk/eval 0.17.8 → 0.18.0

package/dist/cli.js

@@ -2,9 +2,11 @@
 import {
   aggregateRuns,
   evalCompare,
+  evaluateScorerErrorRateGate,
   rescore,
-  runEval
-} from "./chunk-NPKIP4SB.js";
+  runEval,
+  scorerCounts
+} from "./chunk-BWBX5YVQ.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";
@@ -57,10 +59,22 @@ function validateEvalConfig(cfg) {
   if (badScorerIdx !== -1) {
     return `exports a scorers array with a non-scorer entry at index ${badScorerIdx} \u2014 each scorer must have a score() method (use the scorer() factory).`;
   }
+  for (const key of ["concurrency", "scorerConcurrency"]) {
+    const v = c[key];
+    if (v !== void 0 && (typeof v !== "number" || !Number.isInteger(v) || v < 1)) {
+      return `exports an invalid ${key} (${typeof v === "number" ? v : typeof v}) \u2014 must be a positive integer.`;
+    }
+  }
+  {
+    const v = c.failOnScorerErrorRate;
+    if (v !== void 0 && (typeof v !== "number" || !Number.isFinite(v) || v < 0 || v > 1)) {
+      return `exports an invalid failOnScorerErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1.`;
+    }
+  }
   return void 0;
 }
 
-// src/cli.ts
+// src/cli-args.ts
 var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--output",
   "--config",
@@ -68,8 +82,113 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--fail-on-regression",
   "--threshold",
   "--runs",
-  "--capture-traces"
+  "--capture-traces",
+  "--concurrency",
+  "--scorers"
+]);
+var VALUE_FLAGS = /* @__PURE__ */ new Set([
+  "--output",
+  "--config",
+  "--conditions",
+  "--runs",
+  "--concurrency",
+  "--scorers"
 ]);
+function envInt(name) {
+  const raw = process.env[name];
+  if (raw == null || raw === "") return void 0;
+  const n = parseInt(raw, 10);
+  if (!Number.isFinite(n) || n <= 0) return void 0;
+  return n;
+}
+function parseEvalArgs(args) {
+  let outputPath;
+  let configArg;
+  let conditions = [];
+  let runs = 1;
+  let captureTraces = false;
+  let concurrency;
+  let scorerNames;
+  const paths = [];
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (VALUE_FLAGS.has(arg)) {
+      if (i + 1 >= args.length) {
+        console.error(`Error: ${arg} requires a value`);
+        process.exit(1);
+      }
+      const value = args[++i];
+      if (arg === "--output") outputPath = value;
+      else if (arg === "--config") configArg = value;
+      else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
+      else if (arg === "--concurrency") {
+        const n = parseInt(value, 10);
+        if (!Number.isFinite(n) || n <= 0) {
+          console.error(`[axl-eval] Ignoring invalid --concurrency "${value}"; using 1.`);
+          concurrency = 1;
+        } else {
+          concurrency = n;
+        }
+      } else if (arg === "--scorers") {
+        scorerNames = [
+          ...new Set(
+            value.split(",").map((s) => s.trim()).filter(Boolean)
+          )
+        ];
+      } else {
+        conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
+      }
+    } else if (arg === "--capture-traces") {
+      captureTraces = true;
+    } else if (arg.startsWith("--")) {
+      if (!KNOWN_FLAGS.has(arg)) {
+        console.error(`Unknown flag: ${arg}`);
+        process.exit(1);
+      }
+    } else {
+      paths.push(arg);
+    }
+  }
+  return {
+    outputPath,
+    configArg,
+    conditions,
+    runs,
+    captureTraces,
+    concurrency,
+    scorerNames,
+    paths
+  };
+}
+
+// src/cli.ts
+function refuseToGate(reason) {
+  console.error(`[axl-eval] Refusing to gate: ${reason}`);
+  process.exit(1);
+}
+function scorerSampleCounts(entry, name, items) {
+  if (entry.scored != null && entry.failed != null) {
+    return { scored: entry.scored, failed: entry.failed };
+  }
+  if (items) return scorerCounts(items, name);
+  return { scored: entry.scored ?? 0, failed: entry.failed ?? 0 };
+}
+function installAbortHandler(controller) {
+  let signalled = false;
+  const onSignal = (sig) => {
+    if (signalled) {
+      console.error(`
+[axl-eval] Second ${sig} \u2014 forcing exit.`);
+      process.exit(130);
+    }
+    signalled = true;
+    console.error(`
+[axl-eval] ${sig} received \u2014 cancelling eval (press again to force exit)...`);
+    controller.abort();
+  };
+  process.on("SIGINT", onSignal);
+  process.on("SIGTERM", onSignal);
+}
 async function main() {
   const args = process.argv.slice(2);
   if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
@@ -80,6 +199,15 @@ Usage:
   axl-eval <path> --output <file>         Save results to JSON
   axl-eval <path> --config <file>         Use config file for runtime
   axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval <path> --concurrency <n>       Override item concurrency (flag > env
+                                          AXL_EVAL_CONCURRENCY > config > 5).
+                                          Max concurrent scorer calls is
+                                          concurrency x scorerConcurrency (the
+                                          per-eval scorerConcurrency defaults to 5).
+  axl-eval <path> --scorers <a,b>         Run only the named scorers (single eval
+                                          file only). Stamps the result as a
+                                          subset so compare won't treat it as a
+                                          full baseline.
   axl-eval <path> --capture-traces        Populate EvalItem.traces on every item
                                           (success + failure). Adds memory
                                           overhead proportional to dataset size
@@ -88,6 +216,12 @@ Usage:
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+  axl-eval compare <a> <b> --max-scorer-error-rate <0..1>  Exit 1 if a scorer's
+                                          failure rate exceeds the limit on
+                                          either side (deterministic scorers
+                                          tolerate zero failures). Always warns
+                                          when either side rests on a thinned
+                                          sample, even without this flag.
 
 Config auto-detection (when --config is not specified):
   ${CONFIG_CANDIDATES.join(" -> ")}
@@ -102,11 +236,13 @@ When no config is found, a bare AxlRuntime is created (providers from env vars).
     await runCompare(args.slice(1));
     return;
   }
+  const controller = new AbortController();
+  installAbortHandler(controller);
   if (args[0] === "rescore") {
-    await runRescore(args.slice(1));
+    await runRescore(args.slice(1), controller.signal);
     return;
   }
-  await runEvalCommand(args);
+  await runEvalCommand(args, controller.signal);
 }
 function parseThresholdArg(args) {
   const idx = args.indexOf("--threshold");
@@ -120,20 +256,61 @@ function parseThresholdArg(args) {
   }
   return Object.keys(map).length > 0 ? map : void 0;
 }
+function parseMaxScorerErrorRate(args) {
+  const idx = args.indexOf("--max-scorer-error-rate");
+  if (idx === -1) return void 0;
+  if (idx + 1 >= args.length) {
+    console.error("Error: --max-scorer-error-rate requires a value in [0, 1]");
+    process.exit(1);
+  }
+  const raw = args[idx + 1];
+  const n = /^\d*\.?\d+$/.test(raw.trim()) ? Number(raw) : NaN;
+  if (!Number.isFinite(n) || n < 0 || n > 1) {
+    console.error(`Error: --max-scorer-error-rate must be a number in [0, 1], got "${raw}"`);
+    process.exit(1);
+  }
+  return n;
+}
+function scorerFilteredScorers(r) {
+  const first = Array.isArray(r) ? r[0] : r;
+  const meta = first?.metadata;
+  if (!meta || meta.scorerFiltered !== true) return void 0;
+  return Array.isArray(meta.scorersRun) ? meta.scorersRun : [];
+}
 async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
   const thresholds = parseThresholdArg(args);
+  const maxScorerErrorRate = parseMaxScorerErrorRate(args);
+  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate"]);
   const files = args.filter(
-    (a, i) => !a.startsWith("--") && !(i > 0 && args[i - 1] === "--threshold")
+    (a, i) => !a.startsWith("--") && !(i > 0 && valueFlags.has(args[i - 1]))
   );
   if (files.length !== 2) {
-    console.error("Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]");
+    console.error(
+      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>]"
+    );
     process.exit(1);
   }
   const baselineRaw = JSON.parse(await readFileAsync(files[0], "utf-8"));
   const candidateRaw = JSON.parse(await readFileAsync(files[1], "utf-8"));
   const baseline = baselineRaw;
   const candidate = candidateRaw;
+  const filteredSides = [
+    ["baseline", baseline],
+    ["candidate", candidate]
+  ].filter(([, r]) => scorerFilteredScorers(r) != null);
+  for (const [side, r] of filteredSides) {
+    const ran = scorerFilteredScorers(r);
+    const ranStr = ran && ran.length ? ` (ran: ${ran.join(", ")})` : "";
+    console.error(
+      `[axl-eval] WARNING: ${side} was run with --scorers \u2014 it tested only a subset${ranStr}, so this comparison is incomplete.`
+    );
+  }
+  if (failOnRegression && filteredSides.length > 0) {
+    refuseToGate(
+      `${filteredSides.map(([s]) => s).join(", ")} was run with --scorers (a scorer subset); re-run without --scorers for a complete baseline.`
+    );
+  }
   const compareOptions = thresholds != null ? { thresholds } : void 0;
   const comparison = evalCompare(baseline, candidate, compareOptions);
   console.log(
@@ -189,6 +366,29 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}
 `
   );
+  let sawThinnedSample = false;
+  for (const name of scorerNames) {
+    const s = comparison.scorers[name];
+    const bFailed = s.baselineFailed ?? 0;
+    const cFailed = s.candidateFailed ?? 0;
+    if (bFailed > 0 || cFailed > 0) {
+      sawThinnedSample = true;
+      const bAtt = (s.baselineScored ?? 0) + bFailed;
+      const cAtt = (s.candidateScored ?? 0) + cFailed;
+      console.error(
+        `[axl-eval] WARNING: scorer "${name}" failed on some items (baseline ${bFailed}/${bAtt}, candidate ${cFailed}/${cAtt}); the means above are computed over the surviving sample.`
+      );
+    }
+  }
+  if (sawThinnedSample && maxScorerErrorRate == null) {
+    console.error(
+      `[axl-eval] (advisory \u2014 pass --max-scorer-error-rate <0..1> to fail CI on a thinned sample)`
+    );
+  }
+  if (maxScorerErrorRate != null) {
+    const reason = evaluateScorerErrorRateGate(comparison, maxScorerErrorRate);
+    if (reason) refuseToGate(reason);
+  }
   if (failOnRegression && comparison.regressions.length > 0) {
     const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);
     if (hasSignificance) {
@@ -201,12 +401,19 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
     }
   }
 }
-async function runRescore(args) {
-  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);
+async function runRescore(args, signal) {
+  const { outputPath, configArg, conditions, concurrency, scorerNames, paths } = parseEvalArgs(args);
   if (paths.length < 2) {
     console.error("Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]");
     process.exit(1);
   }
+  if (scorerNames?.length) {
+    console.error(
+      "Error: --scorers is not supported with rescore; pass a scorer subset in the eval file instead."
+    );
+    process.exit(1);
+  }
+  const itemConcurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY");
   const [resultsPath, evalFilePath] = paths;
   const raw = JSON.parse(await readFileAsync(resultsPath, "utf-8"));
   const results = Array.isArray(raw) ? raw : [raw];
@@ -226,7 +433,13 @@ async function runRescore(args) {
     }
     const rescored = [];
     for (const resultData of results) {
-      rescored.push(await rescore(resultData, evalConfig.scorers, runtime));
+      if (signal.aborted) break;
+      rescored.push(
+        await rescore(resultData, evalConfig.scorers, runtime, {
+          signal,
+          ...itemConcurrency != null ? { concurrency: itemConcurrency } : {}
+        })
+      );
     }
     for (const r of rescored) {
       console.log("\n" + formatTable(r) + "\n");
@@ -276,14 +489,15 @@ function formatTable(result) {
   lines.push("  " + "\u2500".repeat(ruleLen));
   for (const name of scorerNames) {
     const s = result.summary.scorers[name];
-    const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;
-    if (validScoreCount === 0) {
+    const { scored, failed } = scorerSampleCounts(s, name, result.items);
+    const annotation = failed > 0 ? `   (${scored}/${scored + failed} scored, ${failed} failed)` : "";
+    if (scored === 0) {
       lines.push(
-        `  ${name.padEnd(maxNameLen)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}`
+        `  ${name.padEnd(maxNameLen)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}${annotation}`
       );
     } else {
       lines.push(
-        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`
+        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}${annotation}`
       );
     }
   }
@@ -331,8 +545,10 @@ function formatMultiRunTable(summary) {
   for (const name of scorerNames) {
     const s = summary.scorers[name];
     const meanStd = `${s.mean.toFixed(3)} \xB1 ${s.std.toFixed(3)}`;
+    const { scored, failed } = scorerSampleCounts(s, name);
+    const annotation = failed > 0 ? `   (${scored}/${scored + failed} scored, ${failed} failed)` : "";
     lines.push(
-      `  ${name.padEnd(maxNameLen)}  ${meanStd.padStart(colWidth)}  ${s.min.toFixed(3).padStart(8)}  ${s.max.toFixed(3).padStart(8)}`
+      `  ${name.padEnd(maxNameLen)}  ${meanStd.padStart(colWidth)}  ${s.min.toFixed(3).padStart(8)}  ${s.max.toFixed(3).padStart(8)}${annotation}`
     );
   }
   if (summary.timing) {
@@ -346,6 +562,26 @@ function formatMultiRunTable(summary) {
   lines.push(`  Total Cost: ${costStr} | Total Duration: ${durationStr}s`);
   return lines.join("\n");
 }
+function reportDegraded(result, label) {
+  const d = result.summary.degraded;
+  if (!d || d.length === 0) return false;
+  for (const x of d) {
+    const attempted = x.scored + x.failed;
+    const limitStr = x.type === "deterministic" ? "deterministic zero-tolerance" : `${(x.limit * 100).toFixed(1)}% limit`;
+    console.error(
+      `[axl-eval] DEGRADED: ${label} scorer "${x.scorer}" (${x.type}) failed ${(x.rate * 100).toFixed(1)}% (${x.failed}/${attempted}), over the ${limitStr}.`
+    );
+  }
+  return true;
+}
+function reportTotalWipeout(result, label) {
+  const { count, failures } = result.summary;
+  if (count === 0 || failures < count) return false;
+  console.error(
+    `[axl-eval] FAILED: ${label} \u2014 all ${count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
+  );
+  return true;
+}
 async function resolveRuntimeFromConfig(configPath) {
   try {
     const mod = await importModule(configPath, import.meta.url);
@@ -406,41 +642,17 @@ async function getRuntime(configArg, conditions) {
   const { AxlRuntime } = await import("@axlsdk/axl");
   return new AxlRuntime();
 }
-function parseEvalArgs(args) {
-  let outputPath;
-  let configArg;
-  let conditions = [];
-  let runs = 1;
-  let captureTraces = false;
-  const paths = [];
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-    if (arg === "--output" || arg === "--config" || arg === "--conditions" || arg === "--runs") {
-      if (i + 1 >= args.length) {
-        console.error(`Error: ${arg} requires a value`);
-        process.exit(1);
-      }
-      const value = args[++i];
-      if (arg === "--output") outputPath = value;
-      else if (arg === "--config") configArg = value;
-      else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
-      else
-        conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
-    } else if (arg === "--capture-traces") {
-      captureTraces = true;
-    } else if (arg.startsWith("--")) {
-      if (!KNOWN_FLAGS.has(arg)) {
-        console.error(`Unknown flag: ${arg}`);
-        process.exit(1);
-      }
-    } else {
-      paths.push(arg);
-    }
-  }
-  return { outputPath, configArg, conditions, runs, captureTraces, paths };
-}
-async function runEvalCommand(args) {
-  const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);
+async function runEvalCommand(args, signal) {
+  const {
+    outputPath,
+    configArg,
+    conditions,
+    runs,
+    captureTraces,
+    concurrency,
+    scorerNames,
+    paths
+  } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -453,11 +665,16 @@ async function runEvalCommand(args) {
     console.error("Error: No eval files found");
     process.exit(1);
   }
+  if (scorerNames?.length && evalFiles.length > 1) {
+    console.error(`Error: --scorers requires a single eval file (matched ${evalFiles.length}).`);
+    process.exit(1);
+  }
   const runtime = await getRuntime(configArg, conditions);
   const results = [];
   let failedFiles = 0;
   try {
     for (const filePath of evalFiles) {
+      if (signal.aborted) break;
       try {
         const mod = await importModule(path.resolve(filePath), import.meta.url);
         const evalConfig = pickDefault(mod);
@@ -467,6 +684,22 @@ async function runEvalCommand(args) {
           failedFiles++;
           continue;
         }
+        evalConfig.concurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY") ?? evalConfig.concurrency ?? 5;
+        if (scorerNames?.length) {
+          const available = evalConfig.scorers.map((s) => s.name);
+          const unknown = scorerNames.filter((n) => !available.includes(n));
+          if (unknown.length) {
+            console.error(
+              `Error: --scorers: unknown scorer(s): ${unknown.join(", ")}. Available: ${available.join(", ")}`
+            );
+            process.exit(1);
+          }
+          evalConfig.scorers = evalConfig.scorers.filter((s) => scorerNames.includes(s.name));
+          if (evalConfig.scorers.length === 0) {
+            console.error("Error: --scorers matched no scorers.");
+            process.exit(1);
+          }
+        }
         const exported = pickExport(mod, "executeWorkflow");
         let customExecute;
         if (typeof exported === "function") {
@@ -522,18 +755,27 @@ async function runEvalCommand(args) {
           failedFiles++;
           continue;
         }
-        const runOptions = captureTraces ? { captureTraces: true } : void 0;
+        const runOptions = { signal, ...captureTraces ? { captureTraces: true } : {} };
+        const scorersRun = scorerNames?.length ? evalConfig.scorers.map((s) => s.name) : void 0;
+        const stampFiltered = (result) => {
+          if (scorersRun) {
+            result.metadata.scorerFiltered = true;
+            result.metadata.scorersRun = scorersRun;
+          }
+        };
         if (runs > 1) {
           const { randomUUID } = await import("crypto");
           const runGroupId = randomUUID();
           const runResults = [];
           let runFailure;
           for (let r = 0; r < runs; r++) {
+            if (signal.aborted) break;
             console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
             try {
               const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
               result.metadata.runGroupId = runGroupId;
               result.metadata.runIndex = r;
+              stampFiltered(result);
               runResults.push(result);
             } catch (err) {
               runFailure = err instanceof Error ? err : new Error(String(err));
@@ -565,10 +807,21 @@ async function runEvalCommand(args) {
           const summary = aggregateRuns(runResults);
           console.log("\n" + formatMultiRunTable(summary) + "\n");
           for (const r of runResults) results.push(r);
+          let anyFailing = false;
+          for (const r of runResults) {
+            const wipeout = reportTotalWipeout(r, filePath);
+            const degraded = reportDegraded(r, filePath);
+            if (wipeout || degraded) anyFailing = true;
+          }
+          if (anyFailing && !partial) failedFiles++;
         } else {
           const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
+          stampFiltered(result);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
+          const wipeout = reportTotalWipeout(result, filePath);
+          const degraded = reportDegraded(result, filePath);
+          if (wipeout || degraded) failedFiles++;
         }
       } catch (err) {
         console.error(