@axlsdk/eval 0.17.9 → 0.18.1

package/dist/cli.cjs

@@ -94,10 +94,54 @@ function xorshift32(seed) {
   };
 }
 
-// src/compare.ts
+// src/utils.ts
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
+}
 function round2(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+function scorerCounts(items, name) {
+  let scored = 0;
+  let failed = 0;
+  let skipped = 0;
+  for (const i of items) {
+    if (i.error) continue;
+    if (i.scores[name] != null) scored++;
+    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;
+    else if (i.scoreDetails?.[name]?.duration != null) failed++;
+  }
+  return { scored, failed, skipped };
+}
+function evaluateScorerTolerance(scored, failed, type, limit) {
+  const attempted = scored + failed;
+  const rate = attempted === 0 ? 0 : failed / attempted;
+  const exceeds = attempted > 0 && (type === "deterministic" ? failed > 0 : rate > limit);
+  return { attempted, rate, exceeds, zeroSample: attempted === 0 };
+}
+async function mapWithConcurrency(items, concurrency, task) {
+  const results = new Array(items.length);
+  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
+  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);
+  let next = 0;
+  async function runNext() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await task(items[i], i);
+    }
+  }
+  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+  return results;
+}
+
+// src/compare.ts
 var DEFAULT_LLM_THRESHOLD = 0.05;
 var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
 var LEGACY_THRESHOLD = 0.1;
@@ -171,19 +215,43 @@ function evalCompare(baseline, candidate, options) {
     const delta = round2(deltaRaw);
     const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
     const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    let baselineScored = 0;
+    let baselineFailed = 0;
+    let baselineSkipped = 0;
+    for (const run of baselineRuns) {
+      const c = scorerCounts(run.items, name);
+      baselineScored += c.scored;
+      baselineFailed += c.failed;
+      baselineSkipped += c.skipped;
+    }
+    let candidateScored = 0;
+    let candidateFailed = 0;
+    let candidateSkipped = 0;
+    for (const run of candidateRuns) {
+      const c = scorerCounts(run.items, name);
+      candidateScored += c.scored;
+      candidateFailed += c.failed;
+      candidateSkipped += c.skipped;
+    }
     const entry = {
       baselineMean: bMean,
       candidateMean: cMean,
       delta,
-      deltaPercent
+      deltaPercent,
+      baselineScored,
+      baselineFailed,
+      candidateScored,
+      candidateFailed,
+      baselineSkipped,
+      candidateSkipped
     };
     const diffs = pairedDiffs[name];
+    entry.n = diffs.length;
     if (diffs.length >= 2) {
       const ci = pairedBootstrapCI(diffs);
       entry.ci = { lower: ci.lower, upper: ci.upper };
       entry.pRegression = ci.pRegression;
       entry.pImprovement = ci.pImprovement;
-      entry.n = diffs.length;
       const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
       entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
     }
@@ -311,25 +379,98 @@ function detectPartial(runs) {
   if (runs.length >= attempted) return void 0;
   return { completed: runs.length, attempted };
 }
+function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
+  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
+  for (const name of Object.keys(comparison.scorers)) {
+    const s = comparison.scorers[name];
+    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
+    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
+    for (const side of ["baseline", "candidate"]) {
+      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
+      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
+      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+      if (verdict.zeroSample) {
+        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
+        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
+      }
+      if (verdict.exceeds) {
+        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
+        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
+      }
+    }
+  }
+  return null;
+}
 
 // src/scorer.ts
 function normalizeScorerResult(result) {
   return typeof result === "number" ? { score: result } : result;
 }
-
-// src/utils.ts
-function computeStats(scores) {
-  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
-  const sorted = [...scores].sort((a, b) => a - b);
-  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  const p50 = sorted[Math.floor(sorted.length * 0.5)];
-  const p95 = sorted[Math.floor(sorted.length * 0.95)];
-  return { mean: round3(mean), min: round3(min), max: round3(max), p50: round3(p50), p95: round3(p95) };
+function extractScorerErrorCost(err) {
+  const v = err && typeof err === "object" ? err.cost : void 0;
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
 }
-function round3(n) {
-  return Math.round(n * 1e3) / 1e3;
+
+// src/score-item.ts
+async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
+  const scoreDetails = item.scoreDetails ??= {};
+  for (const s of scorers) {
+    item.scores[s.name] = null;
+    scoreDetails[s.name] = { score: null };
+  }
+  let itemScorerCost = 0;
+  const scorerErrorsByName = {};
+  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer) => {
+    if (signal?.aborted) return;
+    const scorerStart = Date.now();
+    try {
+      if (scorer.applies && !scorer.applies(item.output, item.input, item.annotations)) {
+        item.scores[scorer.name] = null;
+        scoreDetails[scorer.name] = { score: null, skipped: true };
+        return;
+      }
+      const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);
+      const scorerResult = normalizeScorerResult(raw);
+      if (scorerResult.cost != null) {
+        itemScorerCost += scorerResult.cost;
+      }
+      const scorerDuration = Date.now() - scorerStart;
+      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+        scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
+        item.scores[scorer.name] = null;
+        scoreDetails[scorer.name] = {
+          score: null,
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      } else {
+        item.scores[scorer.name] = round2(scorerResult.score);
+        scoreDetails[scorer.name] = {
+          score: round2(scorerResult.score),
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      }
+    } catch (err) {
+      if (err?.name === "AbortError") return;
+      const errCost = extractScorerErrorCost(err);
+      if (errCost != null) itemScorerCost += errCost;
+      scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      item.scores[scorer.name] = null;
+      scoreDetails[scorer.name] = {
+        score: null,
+        duration: Date.now() - scorerStart,
+        cost: errCost
+      };
+    }
+  });
+  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
+  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
+  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+  return itemScorerCost;
 }
 
 // src/runner.ts
@@ -372,7 +513,9 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
+  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
   const concurrency = config.concurrency ?? 5;
+  const scorerConcurrency = config.scorerConcurrency ?? 5;
   const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
   const scorerContext = {
     resolveProvider: (uri) => {
@@ -461,89 +604,53 @@ async function runEval(config, executeWorkflow, runtime, options) {
     if (budgetLimit != null && totalCost > budgetLimit) {
       budgetExceeded = true;
     }
-    evalItem.scoreDetails = {};
-    let itemScorerCost = 0;
-    for (const scorer of config.scorers) {
-      if (options?.signal?.aborted) break;
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer.score(
-          evalItem.output,
-          item.input,
-          item.annotations,
-          scorerContext
-        );
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-          evalItem.scorerErrors.push(
-            `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          evalItem.scores[scorer.name] = null;
-          evalItem.scoreDetails[scorer.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          evalItem.scores[scorer.name] = round3(scorerResult.score);
-          evalItem.scoreDetails[scorer.name] = {
-            score: round3(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-        evalItem.scorerErrors.push(
-          `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        evalItem.scores[scorer.name] = null;
-        evalItem.scoreDetails[scorer.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      evalItem,
+      config.scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     evalItems[itemIndex] = evalItem;
     options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
-  let index = 0;
-  async function runNext() {
-    while (index < items.length) {
-      const currentIndex = index++;
-      await processItem(items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
-  await Promise.all(workers);
+  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
   const failures = evalItems.filter((i) => i.error).length;
   const scorerNames = config.scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed, skipped } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
   }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const scorerTypes = {};
   for (const s of config.scorers) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
   }
+  let degraded;
+  const rawLimit = config.failOnScorerErrorRate;
+  if (rawLimit != null) {
+    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
+      console.warn(
+        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
+      );
+    } else {
+      for (const name of scorerNames) {
+        const stats = scorerStats[name];
+        const scored = stats.scored ?? 0;
+        const failed = stats.failed ?? 0;
+        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
+        const limit = type === "deterministic" ? 0 : rawLimit;
+        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+        if (verdict.exceeds) {
+          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
+        }
+      }
+    }
+  }
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -602,13 +709,20 @@ async function runEval(config, executeWorkflow, runtime, options) {
       ...config.metadata,
       scorerTypes,
       ...modelsMeta,
-      ...workflowsMeta
+      ...workflowsMeta,
+      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
     duration: Date.now() - startTime,
     items: evalItems,
-    summary: { count: items.length, failures, scorers: scorerStats, timing }
+    summary: {
+      count: items.length,
+      failures,
+      scorers: scorerStats,
+      timing,
+      ...degraded ? { degraded } : {}
+    }
   };
 }
 
@@ -617,6 +731,7 @@ var import_node_crypto2 = require("crypto");
 async function rescore(result, scorers, runtime, options) {
   const startTime = Date.now();
   const concurrency = options?.concurrency ?? 5;
+  const scorerConcurrency = options?.scorerConcurrency ?? 5;
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -660,77 +775,24 @@ async function rescore(result, scorers, runtime, options) {
       scores: {},
       scoreDetails: {}
     };
-    let itemScorerCost = 0;
-    for (const scorer of scorers) {
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!item.scorerErrors) item.scorerErrors = [];
-          item.scorerErrors.push(
-            `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          item.scores[scorer.name] = null;
-          item.scoreDetails[scorer.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          item.scores[scorer.name] = round3(scorerResult.score);
-          item.scoreDetails[scorer.name] = {
-            score: round3(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!item.scorerErrors) item.scorerErrors = [];
-        item.scorerErrors.push(
-          `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        item.scores[scorer.name] = null;
-        item.scoreDetails[scorer.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      item,
+      scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     rescored[itemIndex] = item;
   }
-  let index = 0;
-  async function runNext() {
-    while (index < result.items.length) {
-      const currentIndex = index++;
-      await rescoreItem(result.items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from(
-    { length: Math.min(concurrency, result.items.length) },
-    () => runNext()
-  );
-  await Promise.all(workers);
+  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
   const failures = rescored.filter((i) => i.error).length;
   const scorerNames = scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed, skipped } = scorerCounts(rescored, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
   }
   const scorerTypes = {};
   for (const s of scorers) {
@@ -740,7 +802,11 @@ async function rescore(result, scorers, runtime, options) {
     id: (0, import_node_crypto2.randomUUID)(),
     dataset: result.dataset,
     metadata: (() => {
-      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      const rest = { ...result.metadata };
+      delete rest.runGroupId;
+      delete rest.runIndex;
+      delete rest.scorerFiltered;
+      delete rest.scorersRun;
       const merged = {
         ...rest,
         rescored: true,
@@ -793,15 +859,26 @@ function aggregateRuns(runs) {
   const scorerNames = Object.keys(runs[0].summary.scorers);
   const scorers = {};
   for (const name of scorerNames) {
-    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const contributing = runs.filter((r) => {
+      const sc = r.summary.scorers[name]?.scored;
+      return sc == null || sc > 0;
+    });
+    const meanSource = contributing.length > 0 ? contributing : runs;
+    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);
     const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
     const minMean = Math.min(...means);
     const maxMean = Math.max(...means);
+    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
+    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
+    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);
     scorers[name] = {
-      mean: round3(meanOfMeans),
-      std: round3(std(means, meanOfMeans)),
-      min: round3(minMean),
-      max: round3(maxMean)
+      mean: round2(meanOfMeans),
+      std: round2(std(means, meanOfMeans)),
+      min: round2(minMean),
+      max: round2(maxMean),
+      scored,
+      failed,
+      skipped
     };
   }
   let timing;
@@ -809,8 +886,8 @@ function aggregateRuns(runs) {
   if (timingMeans.length === runs.length && timingMeans.length > 0) {
     const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
     timing = {
-      mean: round3(meanTiming),
-      std: round3(std(timingMeans, meanTiming))
+      mean: round2(meanTiming),
+      std: round2(std(timingMeans, meanTiming))
     };
   }
   return {
@@ -818,8 +895,8 @@ function aggregateRuns(runs) {
     runCount: runs.length,
     workflows,
     dataset,
-    totalCost: round3(totalCost),
-    totalDuration: round3(totalDuration),
+    totalCost: round2(totalCost),
+    totalDuration: round2(totalDuration),
     scorers,
     timing
   };
@@ -861,11 +938,22 @@ function validateEvalConfig(cfg) {
   if (badScorerIdx !== -1) {
     return `exports a scorers array with a non-scorer entry at index ${badScorerIdx} \u2014 each scorer must have a score() method (use the scorer() factory).`;
   }
+  for (const key of ["concurrency", "scorerConcurrency"]) {
+    const v = c[key];
+    if (v !== void 0 && (typeof v !== "number" || !Number.isInteger(v) || v < 1)) {
+      return `exports an invalid ${key} (${typeof v === "number" ? v : typeof v}) \u2014 must be a positive integer.`;
+    }
+  }
+  {
+    const v = c.failOnScorerErrorRate;
+    if (v !== void 0 && (typeof v !== "number" || !Number.isFinite(v) || v < 0 || v > 1)) {
+      return `exports an invalid failOnScorerErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1.`;
+    }
+  }
   return void 0;
 }
 
-// src/cli.ts
-var import_meta = {};
+// src/cli-args.ts
 var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--output",
   "--config",
@@ -873,8 +961,99 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--fail-on-regression",
   "--threshold",
   "--runs",
-  "--capture-traces"
+  "--capture-traces",
+  "--concurrency",
+  "--scorers"
+]);
+var VALUE_FLAGS = /* @__PURE__ */ new Set([
+  "--output",
+  "--config",
+  "--conditions",
+  "--runs",
+  "--concurrency",
+  "--scorers"
 ]);
+function envInt(name) {
+  const raw = process.env[name];
+  if (raw == null || raw === "") return void 0;
+  const n = parseInt(raw, 10);
+  if (!Number.isFinite(n) || n <= 0) return void 0;
+  return n;
+}
+function parseEvalArgs(args) {
+  let outputPath;
+  let configArg;
+  let conditions = [];
+  let runs = 1;
+  let captureTraces = false;
+  let concurrency;
+  let scorerNames;
+  const paths = [];
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (VALUE_FLAGS.has(arg)) {
+      if (i + 1 >= args.length) {
+        console.error(`Error: ${arg} requires a value`);
+        process.exit(1);
+      }
+      const value = args[++i];
+      if (arg === "--output") outputPath = value;
+      else if (arg === "--config") configArg = value;
+      else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
+      else if (arg === "--concurrency") {
+        const n = parseInt(value, 10);
+        if (!Number.isFinite(n) || n <= 0) {
+          console.error(`[axl-eval] Ignoring invalid --concurrency "${value}"; using 1.`);
+          concurrency = 1;
+        } else {
+          concurrency = n;
+        }
+      } else if (arg === "--scorers") {
+        scorerNames = [
+          ...new Set(
+            value.split(",").map((s) => s.trim()).filter(Boolean)
+          )
+        ];
+      } else {
+        conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
+      }
+    } else if (arg === "--capture-traces") {
+      captureTraces = true;
+    } else if (arg.startsWith("--")) {
+      if (!KNOWN_FLAGS.has(arg)) {
+        console.error(`Unknown flag: ${arg}`);
+        process.exit(1);
+      }
+    } else {
+      paths.push(arg);
+    }
+  }
+  return {
+    outputPath,
+    configArg,
+    conditions,
+    runs,
+    captureTraces,
+    concurrency,
+    scorerNames,
+    paths
+  };
+}
+
+// src/cli.ts
+var import_meta = {};
+function refuseToGate(reason) {
+  console.error(`[axl-eval] Refusing to gate: ${reason}`);
+  process.exit(1);
+}
+function scorerSampleCounts(entry, name, items) {
+  if (entry.scored != null && entry.failed != null) {
+    const skipped = entry.skipped ?? (items ? scorerCounts(items, name).skipped : 0);
+    return { scored: entry.scored, failed: entry.failed, skipped };
+  }
+  if (items) return scorerCounts(items, name);
+  return { scored: entry.scored ?? 0, failed: entry.failed ?? 0, skipped: entry.skipped ?? 0 };
+}
 function installAbortHandler(controller) {
   let signalled = false;
   const onSignal = (sig) => {
@@ -901,6 +1080,15 @@ Usage:
   axl-eval <path> --output <file>         Save results to JSON
   axl-eval <path> --config <file>         Use config file for runtime
   axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval <path> --concurrency <n>       Override item concurrency (flag > env
+                                          AXL_EVAL_CONCURRENCY > config > 5).
+                                          Max concurrent scorer calls is
+                                          concurrency x scorerConcurrency (the
+                                          per-eval scorerConcurrency defaults to 5).
+  axl-eval <path> --scorers <a,b>         Run only the named scorers (single eval
+                                          file only). Stamps the result as a
+                                          subset so compare won't treat it as a
+                                          full baseline.
   axl-eval <path> --capture-traces        Populate EvalItem.traces on every item
                                           (success + failure). Adds memory
                                           overhead proportional to dataset size
@@ -909,6 +1097,12 @@ Usage:
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+  axl-eval compare <a> <b> --max-scorer-error-rate <0..1>  Exit 1 if a scorer's
+                                          failure rate exceeds the limit on
+                                          either side (deterministic scorers
+                                          tolerate zero failures). Always warns
+                                          when either side rests on a thinned
+                                          sample, even without this flag.
 
 Config auto-detection (when --config is not specified):
   ${import_axl.CONFIG_CANDIDATES.join(" -> ")}
@@ -943,20 +1137,61 @@ function parseThresholdArg(args) {
   }
   return Object.keys(map).length > 0 ? map : void 0;
 }
+function parseMaxScorerErrorRate(args) {
+  const idx = args.indexOf("--max-scorer-error-rate");
+  if (idx === -1) return void 0;
+  if (idx + 1 >= args.length) {
+    console.error("Error: --max-scorer-error-rate requires a value in [0, 1]");
+    process.exit(1);
+  }
+  const raw = args[idx + 1];
+  const n = /^\d*\.?\d+$/.test(raw.trim()) ? Number(raw) : NaN;
+  if (!Number.isFinite(n) || n < 0 || n > 1) {
+    console.error(`Error: --max-scorer-error-rate must be a number in [0, 1], got "${raw}"`);
+    process.exit(1);
+  }
+  return n;
+}
+function scorerFilteredScorers(r) {
+  const first = Array.isArray(r) ? r[0] : r;
+  const meta = first?.metadata;
+  if (!meta || meta.scorerFiltered !== true) return void 0;
+  return Array.isArray(meta.scorersRun) ? meta.scorersRun : [];
+}
 async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
   const thresholds = parseThresholdArg(args);
+  const maxScorerErrorRate = parseMaxScorerErrorRate(args);
+  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate"]);
   const files = args.filter(
-    (a, i) => !a.startsWith("--") && !(i > 0 && args[i - 1] === "--threshold")
+    (a, i) => !a.startsWith("--") && !(i > 0 && valueFlags.has(args[i - 1]))
   );
   if (files.length !== 2) {
-    console.error("Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]");
+    console.error(
+      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>]"
+    );
     process.exit(1);
   }
   const baselineRaw = JSON.parse(await (0, import_promises.readFile)(files[0], "utf-8"));
   const candidateRaw = JSON.parse(await (0, import_promises.readFile)(files[1], "utf-8"));
   const baseline = baselineRaw;
   const candidate = candidateRaw;
+  const filteredSides = [
+    ["baseline", baseline],
+    ["candidate", candidate]
+  ].filter(([, r]) => scorerFilteredScorers(r) != null);
+  for (const [side, r] of filteredSides) {
+    const ran = scorerFilteredScorers(r);
+    const ranStr = ran && ran.length ? ` (ran: ${ran.join(", ")})` : "";
+    console.error(
+      `[axl-eval] WARNING: ${side} was run with --scorers \u2014 it tested only a subset${ranStr}, so this comparison is incomplete.`
+    );
+  }
+  if (failOnRegression && filteredSides.length > 0) {
+    refuseToGate(
+      `${filteredSides.map(([s]) => s).join(", ")} was run with --scorers (a scorer subset); re-run without --scorers for a complete baseline.`
+    );
+  }
   const compareOptions = thresholds != null ? { thresholds } : void 0;
   const comparison = evalCompare(baseline, candidate, compareOptions);
   console.log(
@@ -1012,6 +1247,39 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}
 `
   );
+  let sawThinnedSample = false;
+  for (const name of scorerNames) {
+    const s = comparison.scorers[name];
+    const bFailed = s.baselineFailed ?? 0;
+    const cFailed = s.candidateFailed ?? 0;
+    if (bFailed > 0 || cFailed > 0) {
+      sawThinnedSample = true;
+      const bAtt = (s.baselineScored ?? 0) + bFailed;
+      const cAtt = (s.candidateScored ?? 0) + cFailed;
+      console.error(
+        `[axl-eval] WARNING: scorer "${name}" failed on some items (baseline ${bFailed}/${bAtt}, candidate ${cFailed}/${cAtt}); the means above are computed over the surviving sample.`
+      );
+    }
+    const bSkipped = s.baselineSkipped ?? 0;
+    const cSkipped = s.candidateSkipped ?? 0;
+    const pairedN = s.n ?? 0;
+    const minScored = Math.min(s.baselineScored ?? 0, s.candidateScored ?? 0);
+    const skipsDiverge = (bSkipped > 0 || cSkipped > 0) && (bSkipped !== cSkipped || pairedN < minScored);
+    if (skipsDiverge) {
+      console.error(
+        `[axl-eval] NOTE: scorer "${name}" applies to different subsets per side (baseline ${bSkipped} N/A, candidate ${cSkipped} N/A; ${pairedN} item(s) scored on both); the two means cover different applicable subsets, so compare the delta with care.`
+      );
+    }
+  }
+  if (sawThinnedSample && maxScorerErrorRate == null) {
+    console.error(
+      `[axl-eval] (advisory \u2014 pass --max-scorer-error-rate <0..1> to fail CI on a thinned sample)`
+    );
+  }
+  if (maxScorerErrorRate != null) {
+    const reason = evaluateScorerErrorRateGate(comparison, maxScorerErrorRate);
+    if (reason) refuseToGate(reason);
+  }
   if (failOnRegression && comparison.regressions.length > 0) {
     const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);
     if (hasSignificance) {
@@ -1025,11 +1293,18 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   }
 }
 async function runRescore(args, signal) {
-  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);
+  const { outputPath, configArg, conditions, concurrency, scorerNames, paths } = parseEvalArgs(args);
   if (paths.length < 2) {
     console.error("Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]");
     process.exit(1);
   }
+  if (scorerNames?.length) {
+    console.error(
+      "Error: --scorers is not supported with rescore; pass a scorer subset in the eval file instead."
+    );
+    process.exit(1);
+  }
+  const itemConcurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY");
   const [resultsPath, evalFilePath] = paths;
   const raw = JSON.parse(await (0, import_promises.readFile)(resultsPath, "utf-8"));
   const results = Array.isArray(raw) ? raw : [raw];
@@ -1050,7 +1325,12 @@ async function runRescore(args, signal) {
     const rescored = [];
     for (const resultData of results) {
       if (signal.aborted) break;
-      rescored.push(await rescore(resultData, evalConfig.scorers, runtime, { signal }));
+      rescored.push(
+        await rescore(resultData, evalConfig.scorers, runtime, {
+          signal,
+          ...itemConcurrency != null ? { concurrency: itemConcurrency } : {}
+        })
+      );
     }
     for (const r of rescored) {
       console.log("\n" + formatTable(r) + "\n");
@@ -1100,14 +1380,18 @@ function formatTable(result) {
   lines.push("  " + "\u2500".repeat(ruleLen));
   for (const name of scorerNames) {
     const s = result.summary.scorers[name];
-    const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;
-    if (validScoreCount === 0) {
+    const { scored, failed, skipped } = scorerSampleCounts(s, name, result.items);
+    const annotationParts = [];
+    if (failed > 0) annotationParts.push(`${scored}/${scored + failed} scored, ${failed} failed`);
+    if (skipped > 0) annotationParts.push(`${skipped} N/A`);
+    const annotation = annotationParts.length > 0 ? `   (${annotationParts.join(", ")})` : "";
+    if (scored === 0) {
       lines.push(
-        `  ${name.padEnd(maxNameLen)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}`
+        `  ${name.padEnd(maxNameLen)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}${annotation}`
       );
     } else {
       lines.push(
-        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`
+        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}${annotation}`
       );
     }
   }
@@ -1155,8 +1439,10 @@ function formatMultiRunTable(summary) {
   for (const name of scorerNames) {
     const s = summary.scorers[name];
     const meanStd = `${s.mean.toFixed(3)} \xB1 ${s.std.toFixed(3)}`;
+    const { scored, failed } = scorerSampleCounts(s, name);
+    const annotation = failed > 0 ? `   (${scored}/${scored + failed} scored, ${failed} failed)` : "";
     lines.push(
-      `  ${name.padEnd(maxNameLen)}  ${meanStd.padStart(colWidth)}  ${s.min.toFixed(3).padStart(8)}  ${s.max.toFixed(3).padStart(8)}`
+      `  ${name.padEnd(maxNameLen)}  ${meanStd.padStart(colWidth)}  ${s.min.toFixed(3).padStart(8)}  ${s.max.toFixed(3).padStart(8)}${annotation}`
     );
   }
   if (summary.timing) {
@@ -1170,6 +1456,36 @@ function formatMultiRunTable(summary) {
   lines.push(`  Total Cost: ${costStr} | Total Duration: ${durationStr}s`);
   return lines.join("\n");
 }
+function reportDegraded(result, label) {
+  const d = result.summary.degraded;
+  if (!d || d.length === 0) return false;
+  for (const x of d) {
+    const attempted = x.scored + x.failed;
+    const limitStr = x.type === "deterministic" ? "deterministic zero-tolerance" : `${(x.limit * 100).toFixed(1)}% limit`;
+    console.error(
+      `[axl-eval] DEGRADED: ${label} scorer "${x.scorer}" (${x.type}) failed ${(x.rate * 100).toFixed(1)}% (${x.failed}/${attempted}), over the ${limitStr}.`
+    );
+  }
+  return true;
+}
+function reportFullySkippedScorers(result, label) {
+  for (const [name, entry] of Object.entries(result.summary.scorers)) {
+    const { scored, failed, skipped } = scorerSampleCounts(entry, name, result.items);
+    if (skipped > 0 && scored + failed === 0) {
+      console.error(
+        `[axl-eval] NOTE: ${label} scorer "${name}" was not applicable to any item (${skipped} N/A, 0 scored) \u2014 its mean is not meaningful. Check the scorer's \`applies\` predicate and dataset if you expected it to run.`
+      );
+    }
+  }
+}
+function reportTotalWipeout(result, label) {
+  const { count, failures } = result.summary;
+  if (count === 0 || failures < count) return false;
+  console.error(
+    `[axl-eval] FAILED: ${label} \u2014 all ${count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
+  );
+  return true;
+}
 async function resolveRuntimeFromConfig(configPath) {
   try {
     const mod = await (0, import_axl.importModule)(configPath, import_meta.url);
@@ -1230,41 +1546,17 @@ async function getRuntime(configArg, conditions) {
   const { AxlRuntime } = await import("@axlsdk/axl");
   return new AxlRuntime();
 }
-function parseEvalArgs(args) {
-  let outputPath;
-  let configArg;
-  let conditions = [];
-  let runs = 1;
-  let captureTraces = false;
-  const paths = [];
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-    if (arg === "--output" || arg === "--config" || arg === "--conditions" || arg === "--runs") {
-      if (i + 1 >= args.length) {
-        console.error(`Error: ${arg} requires a value`);
-        process.exit(1);
-      }
-      const value = args[++i];
-      if (arg === "--output") outputPath = value;
-      else if (arg === "--config") configArg = value;
-      else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
-      else
-        conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
-    } else if (arg === "--capture-traces") {
-      captureTraces = true;
-    } else if (arg.startsWith("--")) {
-      if (!KNOWN_FLAGS.has(arg)) {
-        console.error(`Unknown flag: ${arg}`);
-        process.exit(1);
-      }
-    } else {
-      paths.push(arg);
-    }
-  }
-  return { outputPath, configArg, conditions, runs, captureTraces, paths };
-}
 async function runEvalCommand(args, signal) {
-  const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);
+  const {
+    outputPath,
+    configArg,
+    conditions,
+    runs,
+    captureTraces,
+    concurrency,
+    scorerNames,
+    paths
+  } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -1277,6 +1569,10 @@ async function runEvalCommand(args, signal) {
     console.error("Error: No eval files found");
     process.exit(1);
   }
+  if (scorerNames?.length && evalFiles.length > 1) {
+    console.error(`Error: --scorers requires a single eval file (matched ${evalFiles.length}).`);
+    process.exit(1);
+  }
   const runtime = await getRuntime(configArg, conditions);
   const results = [];
   let failedFiles = 0;
@@ -1292,6 +1588,22 @@ async function runEvalCommand(args, signal) {
           failedFiles++;
           continue;
         }
+        evalConfig.concurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY") ?? evalConfig.concurrency ?? 5;
+        if (scorerNames?.length) {
+          const available = evalConfig.scorers.map((s) => s.name);
+          const unknown = scorerNames.filter((n) => !available.includes(n));
+          if (unknown.length) {
+            console.error(
+              `Error: --scorers: unknown scorer(s): ${unknown.join(", ")}. Available: ${available.join(", ")}`
+            );
+            process.exit(1);
+          }
+          evalConfig.scorers = evalConfig.scorers.filter((s) => scorerNames.includes(s.name));
+          if (evalConfig.scorers.length === 0) {
+            console.error("Error: --scorers matched no scorers.");
+            process.exit(1);
+          }
+        }
         const exported = (0, import_axl.pickExport)(mod, "executeWorkflow");
         let customExecute;
         if (typeof exported === "function") {
@@ -1348,6 +1660,13 @@ async function runEvalCommand(args, signal) {
           continue;
         }
         const runOptions = { signal, ...captureTraces ? { captureTraces: true } : {} };
+        const scorersRun = scorerNames?.length ? evalConfig.scorers.map((s) => s.name) : void 0;
+        const stampFiltered = (result) => {
+          if (scorersRun) {
+            result.metadata.scorerFiltered = true;
+            result.metadata.scorersRun = scorersRun;
+          }
+        };
         if (runs > 1) {
           const { randomUUID: randomUUID4 } = await import("crypto");
           const runGroupId = randomUUID4();
@@ -1360,6 +1679,7 @@ async function runEvalCommand(args, signal) {
               const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
               result.metadata.runGroupId = runGroupId;
               result.metadata.runIndex = r;
+              stampFiltered(result);
               runResults.push(result);
             } catch (err) {
               runFailure = err instanceof Error ? err : new Error(String(err));
@@ -1391,10 +1711,23 @@ async function runEvalCommand(args, signal) {
           const summary = aggregateRuns(runResults);
           console.log("\n" + formatMultiRunTable(summary) + "\n");
           for (const r of runResults) results.push(r);
+          let anyFailing = false;
+          for (const r of runResults) {
+            const wipeout = reportTotalWipeout(r, filePath);
+            const degraded = reportDegraded(r, filePath);
+            reportFullySkippedScorers(r, filePath);
+            if (wipeout || degraded) anyFailing = true;
+          }
+          if (anyFailing && !partial) failedFiles++;
         } else {
           const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
+          stampFiltered(result);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
+          const wipeout = reportTotalWipeout(result, filePath);
+          const degraded = reportDegraded(result, filePath);
+          reportFullySkippedScorers(result, filePath);
+          if (wipeout || degraded) failedFiles++;
         }
       } catch (err) {
         console.error(