@axlsdk/eval 0.17.8 → 0.18.0

package/dist/index.cjs

@@ -34,6 +34,8 @@ __export(index_exports, {
   dataset: () => dataset,
   defineEval: () => defineEval,
   evalCompare: () => evalCompare,
+  evaluateScorerErrorRateGate: () => evaluateScorerErrorRateGate,
+  evaluateScorerTolerance: () => evaluateScorerTolerance,
   llmScorer: () => llmScorer,
   normalizeScorerResult: () => normalizeScorerResult,
   pairedBootstrapCI: () => pairedBootstrapCI,
@@ -46,11 +48,71 @@ module.exports = __toCommonJS(index_exports);
 // src/dataset.ts
 var import_promises = require("fs/promises");
 var path = __toESM(require("path"), 1);
+function isPlainObject(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+function isObjectSchema(schema) {
+  const shape = schema.shape;
+  return typeof shape === "object" && shape !== null;
+}
+function collectDroppedKeys(raw, parsed, prefix = "") {
+  if (Array.isArray(raw) && Array.isArray(parsed)) {
+    const dropped = [];
+    const n = Math.min(raw.length, parsed.length);
+    for (let i = 0; i < n; i++) {
+      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));
+    }
+    return dropped;
+  }
+  if (isPlainObject(raw) && isPlainObject(parsed)) {
+    const dropped = [];
+    for (const key of Object.keys(raw)) {
+      const path2 = prefix ? `${prefix}.${key}` : key;
+      if (!(key in parsed)) {
+        dropped.push(path2);
+      } else {
+        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path2));
+      }
+    }
+    return dropped;
+  }
+  return [];
+}
 function dataset(config) {
+  const policy = config.onExtraAnnotationKeys ?? "warn";
+  function parseItem(item, droppedAcc) {
+    const input = config.schema.parse(item.input);
+    if (item.annotations == null || !config.annotations) {
+      return { input, annotations: item.annotations };
+    }
+    const annotations = config.annotations.parse(item.annotations);
+    if (policy !== "ignore" && isObjectSchema(config.annotations)) {
+      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);
+    }
+    return { input, annotations };
+  }
+  function reportDropped(dropped) {
+    if (dropped.size === 0) return;
+    const keys = [...dropped].join(", ");
+    if (policy === "error") {
+      throw new Error(
+        `Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will never reach scorers. Add them to the schema, or set onExtraAnnotationKeys: 'ignore' to allow stripping.`
+      );
+    }
+    console.warn(
+      `[axl-eval] Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will not reach scorers (a common cause of no-op scorers). Add them to the schema, set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`
+    );
+  }
+  let lastDroppedAnnotationKeys = [];
   return {
     name: config.name,
     schema: config.schema,
     annotationsSchema: config.annotations,
+    get droppedAnnotationKeys() {
+      return [...lastDroppedAnnotationKeys];
+    },
     async getItems() {
       if (config.items && config.file) {
         throw new Error('Dataset config error: "items" and "file" are mutually exclusive');
@@ -58,19 +120,19 @@ function dataset(config) {
       if (!config.items && !config.file) {
         throw new Error('Dataset config error: either "items" or "file" must be provided');
       }
+      let rawItems;
       if (config.items) {
-        return config.items.map((item) => ({
-          input: config.schema.parse(item.input),
-          annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-        }));
+        rawItems = config.items;
+      } else {
+        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
+        const content = await (0, import_promises.readFile)(resolvedPath, "utf-8");
+        rawItems = JSON.parse(content);
       }
-      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
-      const content = await (0, import_promises.readFile)(resolvedPath, "utf-8");
-      const items = JSON.parse(content);
-      return items.map((item) => ({
-        input: config.schema.parse(item.input),
-        annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-      }));
+      const dropped = /* @__PURE__ */ new Set();
+      const parsed = rawItems.map((item) => parseItem(item, dropped));
+      lastDroppedAnnotationKeys = [...dropped];
+      reportDropped(dropped);
+      return parsed;
     }
   };
 }
@@ -79,6 +141,15 @@ function dataset(config) {
 function normalizeScorerResult(result) {
   return typeof result === "number" ? { score: result } : result;
 }
+function attachScorerErrorCost(err, cost) {
+  if (err && typeof err === "object" && typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
+    err.cost = cost;
+  }
+}
+function extractScorerErrorCost(err) {
+  const v = err && typeof err === "object" ? err.cost : void 0;
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
+}
 function scorer(config) {
   return {
     name: config.name,
@@ -125,7 +196,18 @@ ${JSON.stringify(annotations, null, 2)}` : "",
           { role: "system", content: config.system },
           { role: "user", content: prompt }
         ],
-        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: "json_object" } }
+        {
+          model,
+          temperature: config.temperature ?? 0.2,
+          maxTokens: config.maxTokens,
+          effort: config.effort,
+          thinkingBudget: config.thinkingBudget,
+          includeThoughts: config.includeThoughts,
+          stop: config.stop,
+          providerOptions: config.providerOptions,
+          responseFormat: { type: "json_object" },
+          signal: context.signal
+        }
       );
       const responseCost = response.cost;
       let validated;
@@ -133,16 +215,14 @@ ${JSON.stringify(annotations, null, 2)}` : "",
         const parsed = JSON.parse((0, import_axl.extractJson)(response.content));
         validated = schema.parse(parsed);
       } catch (err) {
-        if (err && typeof err === "object") {
-          err.cost = responseCost;
-        }
+        attachScorerErrorCost(err, responseCost);
         if (err && typeof err === "object" && "issues" in err && Array.isArray(err.issues)) {
           const issues = err.issues;
           const messages = issues.map((i) => `${i.path.length ? `${i.path.join(".")}: ` : ""}${i.message}`).join("; ");
           const error = new Error(
             `LLM scorer "${config.name}" returned an invalid response: ${messages}`
           );
-          error.cost = responseCost;
+          attachScorerErrorCost(error, responseCost);
           throw error;
         }
         throw err;
@@ -176,6 +256,92 @@ function computeStats(scores) {
 function round(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+function scorerCounts(items, name) {
+  let scored = 0;
+  let failed = 0;
+  for (const i of items) {
+    if (i.error) continue;
+    if (i.scores[name] != null) scored++;
+    else if (i.scoreDetails?.[name]?.duration != null) failed++;
+  }
+  return { scored, failed };
+}
+function evaluateScorerTolerance(scored, failed, type, limit) {
+  const attempted = scored + failed;
+  const rate = attempted === 0 ? 0 : failed / attempted;
+  const exceeds = attempted > 0 && (type === "deterministic" ? failed > 0 : rate > limit);
+  return { attempted, rate, exceeds, zeroSample: attempted === 0 };
+}
+async function mapWithConcurrency(items, concurrency, task) {
+  const results = new Array(items.length);
+  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
+  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);
+  let next = 0;
+  async function runNext() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await task(items[i], i);
+    }
+  }
+  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+  return results;
+}
+
+// src/score-item.ts
+async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
+  const scoreDetails = item.scoreDetails ??= {};
+  for (const s of scorers) {
+    item.scores[s.name] = null;
+    scoreDetails[s.name] = { score: null };
+  }
+  let itemScorerCost = 0;
+  const scorerErrorsByName = {};
+  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer2) => {
+    if (signal?.aborted) return;
+    const scorerStart = Date.now();
+    try {
+      const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
+      const scorerResult = normalizeScorerResult(raw);
+      if (scorerResult.cost != null) {
+        itemScorerCost += scorerResult.cost;
+      }
+      const scorerDuration = Date.now() - scorerStart;
+      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
+        item.scores[scorer2.name] = null;
+        scoreDetails[scorer2.name] = {
+          score: null,
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      } else {
+        item.scores[scorer2.name] = round(scorerResult.score);
+        scoreDetails[scorer2.name] = {
+          score: round(scorerResult.score),
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      }
+    } catch (err) {
+      if (err?.name === "AbortError") return;
+      const errCost = extractScorerErrorCost(err);
+      if (errCost != null) itemScorerCost += errCost;
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      item.scores[scorer2.name] = null;
+      scoreDetails[scorer2.name] = {
+        score: null,
+        duration: Date.now() - scorerStart,
+        cost: errCost
+      };
+    }
+  });
+  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
+  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
+  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+  return itemScorerCost;
+}
 
 // src/runner.ts
 var import_node_crypto = require("crypto");
@@ -217,7 +383,9 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
+  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
   const concurrency = config.concurrency ?? 5;
+  const scorerConcurrency = config.scorerConcurrency ?? 5;
   const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
   const scorerContext = {
     resolveProvider: (uri) => {
@@ -227,7 +395,8 @@ async function runEval(config, executeWorkflow, runtime, options) {
         );
       }
       return runtime.resolveProvider(uri);
-    }
+    },
+    signal: options?.signal
   };
   const evalItems = new Array(items.length);
   let totalCost = 0;
@@ -305,89 +474,53 @@ async function runEval(config, executeWorkflow, runtime, options) {
     if (budgetLimit != null && totalCost > budgetLimit) {
       budgetExceeded = true;
     }
-    evalItem.scoreDetails = {};
-    let itemScorerCost = 0;
-    for (const scorer2 of config.scorers) {
-      if (options?.signal?.aborted) break;
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer2.score(
-          evalItem.output,
-          item.input,
-          item.annotations,
-          scorerContext
-        );
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-          evalItem.scorerErrors.push(
-            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          evalItem.scores[scorer2.name] = null;
-          evalItem.scoreDetails[scorer2.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          evalItem.scores[scorer2.name] = round(scorerResult.score);
-          evalItem.scoreDetails[scorer2.name] = {
-            score: round(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-        evalItem.scorerErrors.push(
-          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        evalItem.scores[scorer2.name] = null;
-        evalItem.scoreDetails[scorer2.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      evalItem,
+      config.scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     evalItems[itemIndex] = evalItem;
     options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
-  let index = 0;
-  async function runNext() {
-    while (index < items.length) {
-      const currentIndex = index++;
-      await processItem(items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
-  await Promise.all(workers);
+  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
   const failures = evalItems.filter((i) => i.error).length;
   const scorerNames = config.scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed };
   }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const scorerTypes = {};
   for (const s of config.scorers) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
   }
+  let degraded;
+  const rawLimit = config.failOnScorerErrorRate;
+  if (rawLimit != null) {
+    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
+      console.warn(
+        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
+      );
+    } else {
+      for (const name of scorerNames) {
+        const stats = scorerStats[name];
+        const scored = stats.scored ?? 0;
+        const failed = stats.failed ?? 0;
+        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
+        const limit = type === "deterministic" ? 0 : rawLimit;
+        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+        if (verdict.exceeds) {
+          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
+        }
+      }
+    }
+  }
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -446,13 +579,20 @@ async function runEval(config, executeWorkflow, runtime, options) {
       ...config.metadata,
       scorerTypes,
       ...modelsMeta,
-      ...workflowsMeta
+      ...workflowsMeta,
+      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
     duration: Date.now() - startTime,
     items: evalItems,
-    summary: { count: items.length, failures, scorers: scorerStats, timing }
+    summary: {
+      count: items.length,
+      failures,
+      scorers: scorerStats,
+      timing,
+      ...degraded ? { degraded } : {}
+    }
   };
 }
 
@@ -523,9 +663,6 @@ function xorshift32(seed) {
 }
 
 // src/compare.ts
-function round3(n) {
-  return Math.round(n * 1e3) / 1e3;
-}
 var DEFAULT_LLM_THRESHOLD = 0.05;
 var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
 var LEGACY_THRESHOLD = 0.1;
@@ -594,16 +731,34 @@ function evalCompare(baseline, candidate, options) {
     const bMeanRaw = baselineMeans[name];
     const cMeanRaw = candidateMeans[name];
     const deltaRaw = cMeanRaw - bMeanRaw;
-    const bMean = round3(bMeanRaw);
-    const cMean = round3(cMeanRaw);
-    const delta = round3(deltaRaw);
-    const deltaPercent = bMeanRaw > 0 ? round3(deltaRaw / bMeanRaw * 100) : 0;
+    const bMean = round(bMeanRaw);
+    const cMean = round(cMeanRaw);
+    const delta = round(deltaRaw);
+    const deltaPercent = bMeanRaw > 0 ? round(deltaRaw / bMeanRaw * 100) : 0;
     const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    let baselineScored = 0;
+    let baselineFailed = 0;
+    for (const run of baselineRuns) {
+      const c = scorerCounts(run.items, name);
+      baselineScored += c.scored;
+      baselineFailed += c.failed;
+    }
+    let candidateScored = 0;
+    let candidateFailed = 0;
+    for (const run of candidateRuns) {
+      const c = scorerCounts(run.items, name);
+      candidateScored += c.scored;
+      candidateFailed += c.failed;
+    }
     const entry = {
       baselineMean: bMean,
       candidateMean: cMean,
       delta,
-      deltaPercent
+      deltaPercent,
+      baselineScored,
+      baselineFailed,
+      candidateScored,
+      candidateFailed
     };
     const diffs = pairedDiffs[name];
     if (diffs.length >= 2) {
@@ -633,15 +788,15 @@ function evalCompare(baseline, candidate, options) {
       if (bScores.length === 0 || cScores.length === 0) continue;
       const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
       const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
-      const delta = round3(cAvg - bAvg);
+      const delta = round(cAvg - bAvg);
       const threshold = resolveThreshold(name, options, baselineRef.metadata);
       if (delta < -threshold)
         regressions.push({
           itemIndex: i,
           input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: round3(bAvg),
-          candidateScore: round3(cAvg),
+          baselineScore: round(bAvg),
+          candidateScore: round(cAvg),
           delta
         });
       else if (delta > threshold)
@@ -649,8 +804,8 @@ function evalCompare(baseline, candidate, options) {
           itemIndex: i,
           input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: round3(bAvg),
-          candidateScore: round3(cAvg),
+          baselineScore: round(bAvg),
+          candidateScore: round(cAvg),
           delta
         });
     }
@@ -661,20 +816,20 @@ function evalCompare(baseline, candidate, options) {
   if (baselineTimings.length > 0 && candidateTimings.length > 0) {
     const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
     const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
-    const delta = round3(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round3((cMean - bMean) / bMean * 100) : 0;
-    timing = { baselineMean: round3(bMean), candidateMean: round3(cMean), delta, deltaPercent };
+    const delta = round(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round((cMean - bMean) / bMean * 100) : 0;
+    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };
   }
   let cost;
   const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
   const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
   if (baselineAvgCost > 0 || candidateAvgCost > 0) {
     const deltaRaw = candidateAvgCost - baselineAvgCost;
-    const delta = round3(deltaRaw);
-    const deltaPercent = baselineAvgCost > 0 ? round3(deltaRaw / baselineAvgCost * 100) : 0;
+    const delta = round(deltaRaw);
+    const deltaPercent = baselineAvgCost > 0 ? round(deltaRaw / baselineAvgCost * 100) : 0;
     cost = {
-      baselineTotal: round3(baselineAvgCost),
-      candidateTotal: round3(candidateAvgCost),
+      baselineTotal: round(baselineAvgCost),
+      candidateTotal: round(candidateAvgCost),
       delta,
       deltaPercent
     };
@@ -739,12 +894,34 @@ function detectPartial(runs) {
   if (runs.length >= attempted) return void 0;
   return { completed: runs.length, attempted };
 }
+function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
+  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
+  for (const name of Object.keys(comparison.scorers)) {
+    const s = comparison.scorers[name];
+    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
+    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
+    for (const side of ["baseline", "candidate"]) {
+      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
+      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+      if (verdict.zeroSample) {
+        return `scorer "${name}" has no scored items on ${side} (its items errored or were skipped) \u2014 cannot certify a mean over an empty sample.`;
+      }
+      if (verdict.exceeds) {
+        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
+        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
+      }
+    }
+  }
+  return null;
+}
 
 // src/rescore.ts
 var import_node_crypto2 = require("crypto");
 async function rescore(result, scorers, runtime, options) {
   const startTime = Date.now();
   const concurrency = options?.concurrency ?? 5;
+  const scorerConcurrency = options?.scorerConcurrency ?? 5;
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -753,11 +930,22 @@ async function rescore(result, scorers, runtime, options) {
         );
       }
       return runtime.resolveProvider(uri);
-    }
+    },
+    signal: options?.signal
   };
   const rescored = new Array(result.items.length);
   let totalCost = 0;
   async function rescoreItem(original, itemIndex) {
+    if (options?.signal?.aborted) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: "cancelled",
+        scores: {}
+      };
+      return;
+    }
     if (original.error) {
       rescored[itemIndex] = {
         input: original.input,
@@ -777,77 +965,24 @@ async function rescore(result, scorers, runtime, options) {
       scores: {},
       scoreDetails: {}
     };
-    let itemScorerCost = 0;
-    for (const scorer2 of scorers) {
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!item.scorerErrors) item.scorerErrors = [];
-          item.scorerErrors.push(
-            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          item.scores[scorer2.name] = null;
-          item.scoreDetails[scorer2.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          item.scores[scorer2.name] = round(scorerResult.score);
-          item.scoreDetails[scorer2.name] = {
-            score: round(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!item.scorerErrors) item.scorerErrors = [];
-        item.scorerErrors.push(
-          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        item.scores[scorer2.name] = null;
-        item.scoreDetails[scorer2.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      item,
+      scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     rescored[itemIndex] = item;
   }
-  let index = 0;
-  async function runNext() {
-    while (index < result.items.length) {
-      const currentIndex = index++;
-      await rescoreItem(result.items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from(
-    { length: Math.min(concurrency, result.items.length) },
-    () => runNext()
-  );
-  await Promise.all(workers);
+  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
   const failures = rescored.filter((i) => i.error).length;
   const scorerNames = scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed } = scorerCounts(rescored, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed };
   }
   const scorerTypes = {};
   for (const s of scorers) {
@@ -857,7 +992,11 @@ async function rescore(result, scorers, runtime, options) {
     id: (0, import_node_crypto2.randomUUID)(),
     dataset: result.dataset,
     metadata: (() => {
-      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      const rest = { ...result.metadata };
+      delete rest.runGroupId;
+      delete rest.runIndex;
+      delete rest.scorerFiltered;
+      delete rest.scorersRun;
       const merged = {
         ...rest,
         rescored: true,
@@ -910,15 +1049,24 @@ function aggregateRuns(runs) {
   const scorerNames = Object.keys(runs[0].summary.scorers);
   const scorers = {};
   for (const name of scorerNames) {
-    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const contributing = runs.filter((r) => {
+      const sc = r.summary.scorers[name]?.scored;
+      return sc == null || sc > 0;
+    });
+    const meanSource = contributing.length > 0 ? contributing : runs;
+    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);
     const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
     const minMean = Math.min(...means);
     const maxMean = Math.max(...means);
+    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
+    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
     scorers[name] = {
       mean: round(meanOfMeans),
       std: round(std(means, meanOfMeans)),
       min: round(minMean),
-      max: round(maxMean)
+      max: round(maxMean),
+      scored,
+      failed
     };
   }
   let timing;
@@ -947,6 +1095,8 @@ function aggregateRuns(runs) {
   dataset,
   defineEval,
   evalCompare,
+  evaluateScorerErrorRateGate,
+  evaluateScorerTolerance,
   llmScorer,
   normalizeScorerResult,
   pairedBootstrapCI,