@axlsdk/eval 0.18.0 → 0.18.2

package/dist/cli.cjs

@@ -111,12 +111,14 @@ function round2(n) {
 function scorerCounts(items, name) {
   let scored = 0;
   let failed = 0;
+  let skipped = 0;
   for (const i of items) {
     if (i.error) continue;
     if (i.scores[name] != null) scored++;
+    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;
     else if (i.scoreDetails?.[name]?.duration != null) failed++;
   }
-  return { scored, failed };
+  return { scored, failed, skipped };
 }
 function evaluateScorerTolerance(scored, failed, type, limit) {
   const attempted = scored + failed;
@@ -215,17 +217,21 @@ function evalCompare(baseline, candidate, options) {
     const threshold = resolveThreshold(name, options, baselineRef.metadata);
     let baselineScored = 0;
     let baselineFailed = 0;
+    let baselineSkipped = 0;
     for (const run of baselineRuns) {
       const c = scorerCounts(run.items, name);
       baselineScored += c.scored;
       baselineFailed += c.failed;
+      baselineSkipped += c.skipped;
     }
     let candidateScored = 0;
     let candidateFailed = 0;
+    let candidateSkipped = 0;
     for (const run of candidateRuns) {
       const c = scorerCounts(run.items, name);
       candidateScored += c.scored;
       candidateFailed += c.failed;
+      candidateSkipped += c.skipped;
     }
     const entry = {
       baselineMean: bMean,
@@ -235,15 +241,17 @@ function evalCompare(baseline, candidate, options) {
       baselineScored,
       baselineFailed,
       candidateScored,
-      candidateFailed
+      candidateFailed,
+      baselineSkipped,
+      candidateSkipped
     };
     const diffs = pairedDiffs[name];
+    entry.n = diffs.length;
     if (diffs.length >= 2) {
       const ci = pairedBootstrapCI(diffs);
       entry.ci = { lower: ci.lower, upper: ci.upper };
       entry.pRegression = ci.pRegression;
       entry.pImprovement = ci.pImprovement;
-      entry.n = diffs.length;
       const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
       entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
     }
@@ -380,9 +388,11 @@ function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
     for (const side of ["baseline", "candidate"]) {
       const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
       const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
       const verdict = evaluateScorerTolerance(scored, failed, type, limit);
       if (verdict.zeroSample) {
-        return `scorer "${name}" has no scored items on ${side} (its items errored or were skipped) \u2014 cannot certify a mean over an empty sample.`;
+        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
+        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
       }
       if (verdict.exceeds) {
         const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
@@ -415,6 +425,11 @@ async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal
     if (signal?.aborted) return;
     const scorerStart = Date.now();
     try {
+      if (scorer.applies && !scorer.applies(item.output, item.input, item.annotations)) {
+        item.scores[scorer.name] = null;
+        scoreDetails[scorer.name] = { score: null, skipped: true };
+        return;
+      }
       const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);
       const scorerResult = normalizeScorerResult(raw);
       if (scorerResult.cost != null) {
@@ -606,8 +621,8 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    const { scored, failed } = scorerCounts(evalItems, name);
-    scorerStats[name] = { ...computeStats(scores), scored, failed };
+    const { scored, failed, skipped } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
   }
   const scorerTypes = {};
   for (const s of config.scorers) {
@@ -776,8 +791,8 @@ async function rescore(result, scorers, runtime, options) {
   const scorerStats = {};
   for (const name of scorerNames) {
     const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    const { scored, failed } = scorerCounts(rescored, name);
-    scorerStats[name] = { ...computeStats(scores), scored, failed };
+    const { scored, failed, skipped } = scorerCounts(rescored, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
   }
   const scorerTypes = {};
   for (const s of scorers) {
@@ -855,13 +870,15 @@ function aggregateRuns(runs) {
     const maxMean = Math.max(...means);
     const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
     const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
+    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);
     scorers[name] = {
       mean: round2(meanOfMeans),
       std: round2(std(means, meanOfMeans)),
       min: round2(minMean),
       max: round2(maxMean),
       scored,
-      failed
+      failed,
+      skipped
     };
   }
   let timing;
@@ -1031,10 +1048,11 @@ function refuseToGate(reason) {
 }
 function scorerSampleCounts(entry, name, items) {
   if (entry.scored != null && entry.failed != null) {
-    return { scored: entry.scored, failed: entry.failed };
+    const skipped = entry.skipped ?? (items ? scorerCounts(items, name).skipped : 0);
+    return { scored: entry.scored, failed: entry.failed, skipped };
   }
   if (items) return scorerCounts(items, name);
-  return { scored: entry.scored ?? 0, failed: entry.failed ?? 0 };
+  return { scored: entry.scored ?? 0, failed: entry.failed ?? 0, skipped: entry.skipped ?? 0 };
 }
 function installAbortHandler(controller) {
   let signalled = false;
@@ -1242,6 +1260,16 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
         `[axl-eval] WARNING: scorer "${name}" failed on some items (baseline ${bFailed}/${bAtt}, candidate ${cFailed}/${cAtt}); the means above are computed over the surviving sample.`
       );
     }
+    const bSkipped = s.baselineSkipped ?? 0;
+    const cSkipped = s.candidateSkipped ?? 0;
+    const pairedN = s.n ?? 0;
+    const minScored = Math.min(s.baselineScored ?? 0, s.candidateScored ?? 0);
+    const skipsDiverge = (bSkipped > 0 || cSkipped > 0) && (bSkipped !== cSkipped || pairedN < minScored);
+    if (skipsDiverge) {
+      console.error(
+        `[axl-eval] NOTE: scorer "${name}" applies to different subsets per side (baseline ${bSkipped} N/A, candidate ${cSkipped} N/A; ${pairedN} item(s) scored on both); the two means cover different applicable subsets, so compare the delta with care.`
+      );
+    }
   }
   if (sawThinnedSample && maxScorerErrorRate == null) {
     console.error(
@@ -1352,8 +1380,11 @@ function formatTable(result) {
   lines.push("  " + "\u2500".repeat(ruleLen));
   for (const name of scorerNames) {
     const s = result.summary.scorers[name];
-    const { scored, failed } = scorerSampleCounts(s, name, result.items);
-    const annotation = failed > 0 ? `   (${scored}/${scored + failed} scored, ${failed} failed)` : "";
+    const { scored, failed, skipped } = scorerSampleCounts(s, name, result.items);
+    const annotationParts = [];
+    if (failed > 0) annotationParts.push(`${scored}/${scored + failed} scored, ${failed} failed`);
+    if (skipped > 0) annotationParts.push(`${skipped} N/A`);
+    const annotation = annotationParts.length > 0 ? `   (${annotationParts.join(", ")})` : "";
     if (scored === 0) {
       lines.push(
         `  ${name.padEnd(maxNameLen)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}  ${"--".padStart(colWidth)}${annotation}`
@@ -1437,6 +1468,16 @@ function reportDegraded(result, label) {
   }
   return true;
 }
+function reportFullySkippedScorers(result, label) {
+  for (const [name, entry] of Object.entries(result.summary.scorers)) {
+    const { scored, failed, skipped } = scorerSampleCounts(entry, name, result.items);
+    if (skipped > 0 && scored + failed === 0) {
+      console.error(
+        `[axl-eval] NOTE: ${label} scorer "${name}" was not applicable to any item (${skipped} N/A, 0 scored) \u2014 its mean is not meaningful. Check the scorer's \`applies\` predicate and dataset if you expected it to run.`
+      );
+    }
+  }
+}
 function reportTotalWipeout(result, label) {
   const { count, failures } = result.summary;
   if (count === 0 || failures < count) return false;
@@ -1674,6 +1715,7 @@ async function runEvalCommand(args, signal) {
           for (const r of runResults) {
             const wipeout = reportTotalWipeout(r, filePath);
             const degraded = reportDegraded(r, filePath);
+            reportFullySkippedScorers(r, filePath);
             if (wipeout || degraded) anyFailing = true;
           }
           if (anyFailing && !partial) failedFiles++;
@@ -1684,6 +1726,7 @@ async function runEvalCommand(args, signal) {
           console.log("\n" + formatTable(result) + "\n");
           const wipeout = reportTotalWipeout(result, filePath);
           const degraded = reportDegraded(result, filePath);
+          reportFullySkippedScorers(result, filePath);
           if (wipeout || degraded) failedFiles++;
         }
       } catch (err) {