@axlsdk/eval 0.17.8 → 0.18.0

package/dist/{chunk-NPKIP4SB.js → chunk-BWBX5YVQ.js}

@@ -64,10 +64,52 @@ function xorshift32(seed) {
   };
 }
 
-// src/compare.ts
+// src/utils.ts
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
+}
 function round2(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+function scorerCounts(items, name) {
+  let scored = 0;
+  let failed = 0;
+  for (const i of items) {
+    if (i.error) continue;
+    if (i.scores[name] != null) scored++;
+    else if (i.scoreDetails?.[name]?.duration != null) failed++;
+  }
+  return { scored, failed };
+}
+function evaluateScorerTolerance(scored, failed, type, limit) {
+  const attempted = scored + failed;
+  const rate = attempted === 0 ? 0 : failed / attempted;
+  const exceeds = attempted > 0 && (type === "deterministic" ? failed > 0 : rate > limit);
+  return { attempted, rate, exceeds, zeroSample: attempted === 0 };
+}
+async function mapWithConcurrency(items, concurrency, task) {
+  const results = new Array(items.length);
+  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
+  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);
+  let next = 0;
+  async function runNext() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await task(items[i], i);
+    }
+  }
+  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+  return results;
+}
+
+// src/compare.ts
 var DEFAULT_LLM_THRESHOLD = 0.05;
 var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
 var LEGACY_THRESHOLD = 0.1;
@@ -141,11 +183,29 @@ function evalCompare(baseline, candidate, options) {
     const delta = round2(deltaRaw);
     const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
     const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    let baselineScored = 0;
+    let baselineFailed = 0;
+    for (const run of baselineRuns) {
+      const c = scorerCounts(run.items, name);
+      baselineScored += c.scored;
+      baselineFailed += c.failed;
+    }
+    let candidateScored = 0;
+    let candidateFailed = 0;
+    for (const run of candidateRuns) {
+      const c = scorerCounts(run.items, name);
+      candidateScored += c.scored;
+      candidateFailed += c.failed;
+    }
     const entry = {
       baselineMean: bMean,
       candidateMean: cMean,
       delta,
-      deltaPercent
+      deltaPercent,
+      baselineScored,
+      baselineFailed,
+      candidateScored,
+      candidateFailed
     };
     const diffs = pairedDiffs[name];
     if (diffs.length >= 2) {
@@ -281,11 +341,41 @@ function detectPartial(runs) {
   if (runs.length >= attempted) return void 0;
   return { completed: runs.length, attempted };
 }
+function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
+  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
+  for (const name of Object.keys(comparison.scorers)) {
+    const s = comparison.scorers[name];
+    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
+    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
+    for (const side of ["baseline", "candidate"]) {
+      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
+      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+      if (verdict.zeroSample) {
+        return `scorer "${name}" has no scored items on ${side} (its items errored or were skipped) \u2014 cannot certify a mean over an empty sample.`;
+      }
+      if (verdict.exceeds) {
+        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
+        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
+      }
+    }
+  }
+  return null;
+}
 
 // src/scorer.ts
 function normalizeScorerResult(result) {
   return typeof result === "number" ? { score: result } : result;
 }
+function attachScorerErrorCost(err, cost) {
+  if (err && typeof err === "object" && typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
+    err.cost = cost;
+  }
+}
+function extractScorerErrorCost(err) {
+  const v = err && typeof err === "object" ? err.cost : void 0;
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
+}
 function scorer(config) {
   return {
     name: config.name,
@@ -295,19 +385,60 @@ function scorer(config) {
   };
 }
 
-// src/utils.ts
-function computeStats(scores) {
-  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
-  const sorted = [...scores].sort((a, b) => a - b);
-  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  const p50 = sorted[Math.floor(sorted.length * 0.5)];
-  const p95 = sorted[Math.floor(sorted.length * 0.95)];
-  return { mean: round3(mean), min: round3(min), max: round3(max), p50: round3(p50), p95: round3(p95) };
-}
-function round3(n) {
-  return Math.round(n * 1e3) / 1e3;
+// src/score-item.ts
+async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
+  const scoreDetails = item.scoreDetails ??= {};
+  for (const s of scorers) {
+    item.scores[s.name] = null;
+    scoreDetails[s.name] = { score: null };
+  }
+  let itemScorerCost = 0;
+  const scorerErrorsByName = {};
+  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer2) => {
+    if (signal?.aborted) return;
+    const scorerStart = Date.now();
+    try {
+      const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
+      const scorerResult = normalizeScorerResult(raw);
+      if (scorerResult.cost != null) {
+        itemScorerCost += scorerResult.cost;
+      }
+      const scorerDuration = Date.now() - scorerStart;
+      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
+        item.scores[scorer2.name] = null;
+        scoreDetails[scorer2.name] = {
+          score: null,
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      } else {
+        item.scores[scorer2.name] = round2(scorerResult.score);
+        scoreDetails[scorer2.name] = {
+          score: round2(scorerResult.score),
+          metadata: scorerResult.metadata,
+          duration: scorerDuration,
+          cost: scorerResult.cost
+        };
+      }
+    } catch (err) {
+      if (err?.name === "AbortError") return;
+      const errCost = extractScorerErrorCost(err);
+      if (errCost != null) itemScorerCost += errCost;
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      item.scores[scorer2.name] = null;
+      scoreDetails[scorer2.name] = {
+        score: null,
+        duration: Date.now() - scorerStart,
+        cost: errCost
+      };
+    }
+  });
+  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
+  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
+  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+  return itemScorerCost;
 }
 
 // src/runner.ts
@@ -350,7 +481,9 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = randomUUID();
   const items = await config.dataset.getItems();
+  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
   const concurrency = config.concurrency ?? 5;
+  const scorerConcurrency = config.scorerConcurrency ?? 5;
   const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
   const scorerContext = {
     resolveProvider: (uri) => {
@@ -360,7 +493,8 @@ async function runEval(config, executeWorkflow, runtime, options) {
         );
       }
       return runtime.resolveProvider(uri);
-    }
+    },
+    signal: options?.signal
   };
   const evalItems = new Array(items.length);
   let totalCost = 0;
@@ -438,89 +572,53 @@ async function runEval(config, executeWorkflow, runtime, options) {
     if (budgetLimit != null && totalCost > budgetLimit) {
       budgetExceeded = true;
     }
-    evalItem.scoreDetails = {};
-    let itemScorerCost = 0;
-    for (const scorer2 of config.scorers) {
-      if (options?.signal?.aborted) break;
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer2.score(
-          evalItem.output,
-          item.input,
-          item.annotations,
-          scorerContext
-        );
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-          evalItem.scorerErrors.push(
-            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          evalItem.scores[scorer2.name] = null;
-          evalItem.scoreDetails[scorer2.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          evalItem.scores[scorer2.name] = round3(scorerResult.score);
-          evalItem.scoreDetails[scorer2.name] = {
-            score: round3(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-        evalItem.scorerErrors.push(
-          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        evalItem.scores[scorer2.name] = null;
-        evalItem.scoreDetails[scorer2.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      evalItem,
+      config.scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     evalItems[itemIndex] = evalItem;
     options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
-  let index = 0;
-  async function runNext() {
-    while (index < items.length) {
-      const currentIndex = index++;
-      await processItem(items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
-  await Promise.all(workers);
+  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
   const failures = evalItems.filter((i) => i.error).length;
   const scorerNames = config.scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed };
   }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const scorerTypes = {};
   for (const s of config.scorers) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
   }
+  let degraded;
+  const rawLimit = config.failOnScorerErrorRate;
+  if (rawLimit != null) {
+    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
+      console.warn(
+        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
+      );
+    } else {
+      for (const name of scorerNames) {
+        const stats = scorerStats[name];
+        const scored = stats.scored ?? 0;
+        const failed = stats.failed ?? 0;
+        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
+        const limit = type === "deterministic" ? 0 : rawLimit;
+        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+        if (verdict.exceeds) {
+          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
+        }
+      }
+    }
+  }
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -579,13 +677,20 @@ async function runEval(config, executeWorkflow, runtime, options) {
       ...config.metadata,
       scorerTypes,
       ...modelsMeta,
-      ...workflowsMeta
+      ...workflowsMeta,
+      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
     duration: Date.now() - startTime,
     items: evalItems,
-    summary: { count: items.length, failures, scorers: scorerStats, timing }
+    summary: {
+      count: items.length,
+      failures,
+      scorers: scorerStats,
+      timing,
+      ...degraded ? { degraded } : {}
+    }
   };
 }
 
@@ -594,6 +699,7 @@ import { randomUUID as randomUUID2 } from "crypto";
 async function rescore(result, scorers, runtime, options) {
   const startTime = Date.now();
   const concurrency = options?.concurrency ?? 5;
+  const scorerConcurrency = options?.scorerConcurrency ?? 5;
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -602,11 +708,22 @@ async function rescore(result, scorers, runtime, options) {
         );
       }
       return runtime.resolveProvider(uri);
-    }
+    },
+    signal: options?.signal
   };
   const rescored = new Array(result.items.length);
   let totalCost = 0;
   async function rescoreItem(original, itemIndex) {
+    if (options?.signal?.aborted) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: "cancelled",
+        scores: {}
+      };
+      return;
+    }
     if (original.error) {
       rescored[itemIndex] = {
         input: original.input,
@@ -626,77 +743,24 @@ async function rescore(result, scorers, runtime, options) {
       scores: {},
       scoreDetails: {}
     };
-    let itemScorerCost = 0;
-    for (const scorer2 of scorers) {
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!item.scorerErrors) item.scorerErrors = [];
-          item.scorerErrors.push(
-            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          item.scores[scorer2.name] = null;
-          item.scoreDetails[scorer2.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          item.scores[scorer2.name] = round3(scorerResult.score);
-          item.scoreDetails[scorer2.name] = {
-            score: round3(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!item.scorerErrors) item.scorerErrors = [];
-        item.scorerErrors.push(
-          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        item.scores[scorer2.name] = null;
-        item.scoreDetails[scorer2.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    const itemScorerCost = await scoreItem(
+      item,
+      scorers,
+      scorerConcurrency,
+      scorerContext,
+      options?.signal
+    );
+    totalCost += itemScorerCost;
     rescored[itemIndex] = item;
   }
-  let index = 0;
-  async function runNext() {
-    while (index < result.items.length) {
-      const currentIndex = index++;
-      await rescoreItem(result.items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from(
-    { length: Math.min(concurrency, result.items.length) },
-    () => runNext()
-  );
-  await Promise.all(workers);
+  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
   const failures = rescored.filter((i) => i.error).length;
   const scorerNames = scorers.map((s) => s.name);
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
+    const { scored, failed } = scorerCounts(rescored, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed };
   }
   const scorerTypes = {};
   for (const s of scorers) {
@@ -706,7 +770,11 @@ async function rescore(result, scorers, runtime, options) {
     id: randomUUID2(),
     dataset: result.dataset,
     metadata: (() => {
-      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      const rest = { ...result.metadata };
+      delete rest.runGroupId;
+      delete rest.runIndex;
+      delete rest.scorerFiltered;
+      delete rest.scorersRun;
       const merged = {
         ...rest,
         rescored: true,
@@ -759,15 +827,24 @@ function aggregateRuns(runs) {
   const scorerNames = Object.keys(runs[0].summary.scorers);
   const scorers = {};
   for (const name of scorerNames) {
-    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const contributing = runs.filter((r) => {
+      const sc = r.summary.scorers[name]?.scored;
+      return sc == null || sc > 0;
+    });
+    const meanSource = contributing.length > 0 ? contributing : runs;
+    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);
     const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
     const minMean = Math.min(...means);
     const maxMean = Math.max(...means);
+    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
+    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
     scorers[name] = {
-      mean: round3(meanOfMeans),
-      std: round3(std(means, meanOfMeans)),
-      min: round3(minMean),
-      max: round3(maxMean)
+      mean: round2(meanOfMeans),
+      std: round2(std(means, meanOfMeans)),
+      min: round2(minMean),
+      max: round2(maxMean),
+      scored,
+      failed
     };
   }
   let timing;
@@ -775,8 +852,8 @@ function aggregateRuns(runs) {
   if (timingMeans.length === runs.length && timingMeans.length > 0) {
     const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
     timing = {
-      mean: round3(meanTiming),
-      std: round3(std(timingMeans, meanTiming))
+      mean: round2(meanTiming),
+      std: round2(std(timingMeans, meanTiming))
     };
   }
   return {
@@ -784,8 +861,8 @@ function aggregateRuns(runs) {
     runCount: runs.length,
     workflows,
     dataset,
-    totalCost: round3(totalCost),
-    totalDuration: round3(totalDuration),
+    totalCost: round2(totalCost),
+    totalDuration: round2(totalDuration),
     scorers,
     timing
   };
@@ -793,11 +870,15 @@ function aggregateRuns(runs) {
 
 export {
   pairedBootstrapCI,
+  scorerCounts,
+  evaluateScorerTolerance,
   evalCompare,
+  evaluateScorerErrorRateGate,
   normalizeScorerResult,
+  attachScorerErrorCost,
   scorer,
   runEval,
   rescore,
   aggregateRuns
 };
-//# sourceMappingURL=chunk-NPKIP4SB.js.map
+//# sourceMappingURL=chunk-BWBX5YVQ.js.map