@axlsdk/eval 0.17.9 → 0.18.0

package/dist/index.d.cts

@@ -5,6 +5,27 @@ type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
     annotations?: TAnnotations;
 };
+/**
+ * What to do when a dataset item's `annotations` contain keys that are absent
+ * from the `annotations` schema.
+ *
+ * Zod object schemas strip unknown keys by default, so an annotation field
+ * that isn't declared in the schema is silently dropped before it ever reaches
+ * a scorer. A scorer that reads such a field sees `undefined` and quietly
+ * degrades to a no-op — the eval still "passes", so the bug is invisible. This
+ * is especially easy to hit with file-based datasets, where the JSON is never
+ * type-checked.
+ *
+ * - `'warn'` (default) — log a single `console.warn` per dataset listing the
+ *   dropped key paths. Non-breaking; surfaces the contract violation.
+ * - `'error'` — throw, listing the dropped key paths. The strict path for CI.
+ * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).
+ *
+ * Only `annotations` are checked, not `input`: stripped input fields flow to
+ * the workflow where they surface, whereas stripped annotations vanish into a
+ * silent scorer.
+ */
+type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';
 type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {
     name: string;
     schema: TInput;
@@ -13,11 +34,28 @@ type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.
     file?: string;
     /** Base directory for resolving relative file paths. Defaults to cwd. */
     basePath?: string;
+    /**
+     * How to handle annotation keys not present in the `annotations` schema
+     * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.
+     */
+    onExtraAnnotationKeys?: ExtraKeyPolicy;
 };
 type Dataset<TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly schema: z.ZodType;
     readonly annotationsSchema?: z.ZodType;
+    /**
+     * Annotation key paths the schema dropped during the most recent
+     * `getItems()` call (empty when none were dropped, or when
+     * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of
+     * `getItems()` — read it synchronously right after awaiting, on the same
+     * instance; a later `getItems()` overwrites it (calling `getItems()`
+     * concurrently on one shared dataset is unsupported). The eval runner folds
+     * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the
+     * CLI logs it; UIs can render it) sees the same signal. Optional so
+     * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.
+     */
+    readonly droppedAnnotationKeys?: readonly string[];
     getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
@@ -86,10 +124,53 @@ type EvalConfig = {
     workflow: string;
     dataset: Dataset<unknown, unknown>;
     scorers: Scorer<unknown, unknown, unknown>[];
+    /** Item-level worker-pool size (how many dataset items run in parallel). Default 5. */
     concurrency?: number;
+    /**
+     * Per-item scorer fan-out — how many of an item's scorers run concurrently.
+     * Default 5 (matches `concurrency`; scorers are parallel-by-default). The
+     * worst-case number of concurrent scorer calls is `concurrency × scorerConcurrency`,
+     * so lower `concurrency` if a rate-limited judge model needs a tighter ceiling.
+     */
+    scorerConcurrency?: number;
     budget?: string;
+    /**
+     * Opt-in source-side trust gate. When set (0–1), `runEval` flags the run as
+     * `summary.degraded` if any scorer's failure rate exceeds tolerance, and the
+     * CLI exits non-zero. Enforcement is **type-aware**: deterministic scorers
+     * tolerate ZERO failures (a deterministic scorer that throws is a bug, not
+     * noise), while LLM scorers use this rate (a flaky judge / rate-limit storm).
+     *
+     * Failure rate = `failed / (scored + failed)` per scorer, where `scored` is
+     * the number of items that produced a valid numeric score and `failed` is the
+     * number whose scorer ran and threw / returned out-of-range. `0` means "any
+     * LLM failure degrades the run". Invalid values (non-finite, <0, >1) are
+     * ignored with a `console.warn`.
+     *
+     * Distinct from the gate-side `--max-scorer-error-rate` compare flag: this one
+     * fires at run/produce time (catch a thinned sample before it's saved), the
+     * other at consume/gate time (refuse to certify a thinned baseline/candidate).
+     */
+    failOnScorerErrorRate?: number;
     metadata?: Record<string, unknown>;
 };
+/**
+ * One scorer whose failure rate tripped `EvalConfig.failOnScorerErrorRate`.
+ * Surfaced on `EvalSummary.degraded` so consumers (CLI exit code, Studio
+ * banner) can refuse to trust the mean without re-deriving the rate.
+ */
+type DegradedScorer = {
+    scorer: string;
+    /** Observed failure rate `failed / (scored + failed)` (0 when nothing ran). */
+    rate: number;
+    /** The tolerance that was exceeded (0 for deterministic scorers). */
+    limit: number;
+    type: 'llm' | 'deterministic';
+    /** Items that produced a valid numeric score. */
+    scored: number;
+    /** Items whose scorer ran and failed (threw / out-of-range). */
+    failed: number;
+};
 type EvalResult = {
     id: string;
     /**
@@ -108,6 +189,13 @@ type EvalResult = {
      *   that need a single "primary" workflow should use `workflows[0]`.
      * - `scorerTypes: Record<string, 'llm' | 'deterministic'>`
      * - `runGroupId?: string`, `runIndex?: number` for multi-run groups
+     * - `droppedAnnotationKeys?: string[]` — annotation key paths the dataset
+     *   schema stripped (present only when non-empty). Mirrors the dataset's
+     *   `console.warn`; lets any consumer surface the same signal.
+     * - `scorerFiltered?: boolean` + `scorersRun?: string[]` — set when the CLI
+     *   `--scorers` flag ran a subset of scorers. A guard so a filtered run isn't
+     *   silently used as a full baseline (`compare` warns / refuses to gate on it).
+     *   Distinct from the multi-run "partial" concept (see {@link EvalComparisonPartial}).
      */
     metadata: Record<string, unknown>;
     timestamp: string;
@@ -150,6 +238,22 @@ type EvalSummary = {
         max: number;
         p50: number;
         p95: number;
+        /**
+         * Number of items that produced a valid numeric score — the sample size
+         * the `mean` actually covers. Optional so pre-existing artifacts (and
+         * hand-rolled summaries) stay valid; absent ⇒ recompute from `items`.
+         * Named `scored` (NOT `n`) to avoid collision with
+         * `EvalComparison.scorers[].n` (paired-diff count).
+         */
+        scored?: number;
+        /**
+         * Number of items whose scorer RAN and failed (threw or returned
+         * out-of-range). Distinct from items skipped by cancellation, which land
+         * in neither bucket — so `scored + failed` is the honest "attempted"
+         * denominator and may be `<` the eligible item count. A non-zero `failed`
+         * means the `mean` was computed over a thinned sample.
+         */
+        failed?: number;
     }>;
     timing?: {
         mean: number;
@@ -158,6 +262,14 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /**
+     * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
+     * and one or more scorers exceeded tolerance. `runEval` never throws on this
+     * — it returns the (still-useful) result with this flag set, and the CLI /
+     * consumer decides whether to fail. Absent / empty ⇒ no degradation gate
+     * tripped (either not configured or all scorers within tolerance).
+     */
+    degraded?: DegradedScorer[];
 };
 /**
  * Sample-size context for one side of a comparison.
@@ -216,6 +328,18 @@ type EvalComparison = {
         pImprovement?: number;
         /** Number of paired item differences used for CI computation. */
         n?: number;
+        /**
+         * Per-side scorer success/failure counts over the SAME truncated pool the
+         * means and CI are computed from (not a separately-recomputed raw set), so
+         * a gate reads numbers consistent with what the table displays. Raw counts
+         * (consumer divides) keep this truncation-consistent. A non-zero
+         * `*Failed` means that side's mean rests on a thinned sample — the gate-side
+         * `--max-scorer-error-rate` flag refuses to certify when it's over tolerance.
+         */
+        baselineScored?: number;
+        baselineFailed?: number;
+        candidateScored?: number;
+        candidateFailed?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -297,6 +421,48 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 }>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
+/**
+ * Decide whether a comparison should be REFUSED under a gate-side scorer
+ * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and
+ * testable — returns a human-readable refusal reason, or `null` to allow.
+ *
+ * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY
+ * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a
+ * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`
+ * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the
+ * permissive tier) is deliberate: classifying an old artifact's scorers as
+ * deterministic would hard-fail them on the first flake, contradicting the
+ * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical
+ * across sides (enforced by `evalCompare`), so reading types from the baseline
+ * is sufficient.
+ *
+ * Also refuses a scorer that produced ZERO scored items on a side — a mean over
+ * an empty sample can't be certified. Counts come from the same truncated pool
+ * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).
+ */
+declare function evaluateScorerErrorRateGate(comparison: EvalComparison, maxScorerErrorRate: number): string | null;
+
+/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */
+type ScorerToleranceVerdict = {
+    /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */
+    attempted: number;
+    /** `failed / attempted`, or `0` when nothing was attempted. */
+    rate: number;
+    /** Over tolerance AND something was actually attempted. */
+    exceeds: boolean;
+    /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */
+    zeroSample: boolean;
+};
+/**
+ * Single source of truth for the type-aware failure-rate decision, shared by the
+ * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare
+ * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures
+ * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`
+ * (`failed / attempted`). `exceeds` is never true when nothing was attempted —
+ * callers decide their own policy for `zeroSample` (the runner skips it; the
+ * compare gate refuses to certify a zero-sample scorer).
+ */
+declare function evaluateScorerTolerance(scored: number, failed: number, type: 'llm' | 'deterministic', limit: number): ScorerToleranceVerdict;
 
 type BootstrapCIResult = {
     /** Lower bound of the confidence interval. */
@@ -323,7 +489,12 @@ declare function pairedBootstrapCI(differences: number[], options?: {
 }): BootstrapCIResult;
 
 type RescoreOptions = {
+    /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */
     concurrency?: number;
+    /** Per-item scorer fan-out (how many scorers run concurrently within one item).
+     *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent
+     *  judge calls is `concurrency × scorerConcurrency`. */
+    scorerConcurrency?: number;
     /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can
      *  be cancelled mid-flight. Also checked between items to short-circuit
      *  remaining work. Mirrors `RunEvalOptions.signal`. */
@@ -353,6 +524,10 @@ type MultiRunSummary = {
         std: number;
         min: number;
         max: number;
+        /** Total valid scores summed across all runs in the group. */
+        scored?: number;
+        /** Total ran-and-failed scorer invocations summed across all runs. */
+        failed?: number;
     }>;
     timing?: {
         mean: number;
@@ -364,4 +539,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.d.ts

@@ -5,6 +5,27 @@ type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
     annotations?: TAnnotations;
 };
+/**
+ * What to do when a dataset item's `annotations` contain keys that are absent
+ * from the `annotations` schema.
+ *
+ * Zod object schemas strip unknown keys by default, so an annotation field
+ * that isn't declared in the schema is silently dropped before it ever reaches
+ * a scorer. A scorer that reads such a field sees `undefined` and quietly
+ * degrades to a no-op — the eval still "passes", so the bug is invisible. This
+ * is especially easy to hit with file-based datasets, where the JSON is never
+ * type-checked.
+ *
+ * - `'warn'` (default) — log a single `console.warn` per dataset listing the
+ *   dropped key paths. Non-breaking; surfaces the contract violation.
+ * - `'error'` — throw, listing the dropped key paths. The strict path for CI.
+ * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).
+ *
+ * Only `annotations` are checked, not `input`: stripped input fields flow to
+ * the workflow where they surface, whereas stripped annotations vanish into a
+ * silent scorer.
+ */
+type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';
 type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {
     name: string;
     schema: TInput;
@@ -13,11 +34,28 @@ type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.
     file?: string;
     /** Base directory for resolving relative file paths. Defaults to cwd. */
     basePath?: string;
+    /**
+     * How to handle annotation keys not present in the `annotations` schema
+     * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.
+     */
+    onExtraAnnotationKeys?: ExtraKeyPolicy;
 };
 type Dataset<TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly schema: z.ZodType;
     readonly annotationsSchema?: z.ZodType;
+    /**
+     * Annotation key paths the schema dropped during the most recent
+     * `getItems()` call (empty when none were dropped, or when
+     * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of
+     * `getItems()` — read it synchronously right after awaiting, on the same
+     * instance; a later `getItems()` overwrites it (calling `getItems()`
+     * concurrently on one shared dataset is unsupported). The eval runner folds
+     * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the
+     * CLI logs it; UIs can render it) sees the same signal. Optional so
+     * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.
+     */
+    readonly droppedAnnotationKeys?: readonly string[];
     getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
@@ -86,10 +124,53 @@ type EvalConfig = {
     workflow: string;
     dataset: Dataset<unknown, unknown>;
     scorers: Scorer<unknown, unknown, unknown>[];
+    /** Item-level worker-pool size (how many dataset items run in parallel). Default 5. */
     concurrency?: number;
+    /**
+     * Per-item scorer fan-out — how many of an item's scorers run concurrently.
+     * Default 5 (matches `concurrency`; scorers are parallel-by-default). The
+     * worst-case number of concurrent scorer calls is `concurrency × scorerConcurrency`,
+     * so lower `concurrency` if a rate-limited judge model needs a tighter ceiling.
+     */
+    scorerConcurrency?: number;
     budget?: string;
+    /**
+     * Opt-in source-side trust gate. When set (0–1), `runEval` flags the run as
+     * `summary.degraded` if any scorer's failure rate exceeds tolerance, and the
+     * CLI exits non-zero. Enforcement is **type-aware**: deterministic scorers
+     * tolerate ZERO failures (a deterministic scorer that throws is a bug, not
+     * noise), while LLM scorers use this rate (a flaky judge / rate-limit storm).
+     *
+     * Failure rate = `failed / (scored + failed)` per scorer, where `scored` is
+     * the number of items that produced a valid numeric score and `failed` is the
+     * number whose scorer ran and threw / returned out-of-range. `0` means "any
+     * LLM failure degrades the run". Invalid values (non-finite, <0, >1) are
+     * ignored with a `console.warn`.
+     *
+     * Distinct from the gate-side `--max-scorer-error-rate` compare flag: this one
+     * fires at run/produce time (catch a thinned sample before it's saved), the
+     * other at consume/gate time (refuse to certify a thinned baseline/candidate).
+     */
+    failOnScorerErrorRate?: number;
     metadata?: Record<string, unknown>;
 };
+/**
+ * One scorer whose failure rate tripped `EvalConfig.failOnScorerErrorRate`.
+ * Surfaced on `EvalSummary.degraded` so consumers (CLI exit code, Studio
+ * banner) can refuse to trust the mean without re-deriving the rate.
+ */
+type DegradedScorer = {
+    scorer: string;
+    /** Observed failure rate `failed / (scored + failed)` (0 when nothing ran). */
+    rate: number;
+    /** The tolerance that was exceeded (0 for deterministic scorers). */
+    limit: number;
+    type: 'llm' | 'deterministic';
+    /** Items that produced a valid numeric score. */
+    scored: number;
+    /** Items whose scorer ran and failed (threw / out-of-range). */
+    failed: number;
+};
 type EvalResult = {
     id: string;
     /**
@@ -108,6 +189,13 @@ type EvalResult = {
      *   that need a single "primary" workflow should use `workflows[0]`.
      * - `scorerTypes: Record<string, 'llm' | 'deterministic'>`
      * - `runGroupId?: string`, `runIndex?: number` for multi-run groups
+     * - `droppedAnnotationKeys?: string[]` — annotation key paths the dataset
+     *   schema stripped (present only when non-empty). Mirrors the dataset's
+     *   `console.warn`; lets any consumer surface the same signal.
+     * - `scorerFiltered?: boolean` + `scorersRun?: string[]` — set when the CLI
+     *   `--scorers` flag ran a subset of scorers. A guard so a filtered run isn't
+     *   silently used as a full baseline (`compare` warns / refuses to gate on it).
+     *   Distinct from the multi-run "partial" concept (see {@link EvalComparisonPartial}).
      */
     metadata: Record<string, unknown>;
     timestamp: string;
@@ -150,6 +238,22 @@ type EvalSummary = {
         max: number;
         p50: number;
         p95: number;
+        /**
+         * Number of items that produced a valid numeric score — the sample size
+         * the `mean` actually covers. Optional so pre-existing artifacts (and
+         * hand-rolled summaries) stay valid; absent ⇒ recompute from `items`.
+         * Named `scored` (NOT `n`) to avoid collision with
+         * `EvalComparison.scorers[].n` (paired-diff count).
+         */
+        scored?: number;
+        /**
+         * Number of items whose scorer RAN and failed (threw or returned
+         * out-of-range). Distinct from items skipped by cancellation, which land
+         * in neither bucket — so `scored + failed` is the honest "attempted"
+         * denominator and may be `<` the eligible item count. A non-zero `failed`
+         * means the `mean` was computed over a thinned sample.
+         */
+        failed?: number;
     }>;
     timing?: {
         mean: number;
@@ -158,6 +262,14 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /**
+     * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
+     * and one or more scorers exceeded tolerance. `runEval` never throws on this
+     * — it returns the (still-useful) result with this flag set, and the CLI /
+     * consumer decides whether to fail. Absent / empty ⇒ no degradation gate
+     * tripped (either not configured or all scorers within tolerance).
+     */
+    degraded?: DegradedScorer[];
 };
 /**
  * Sample-size context for one side of a comparison.
@@ -216,6 +328,18 @@ type EvalComparison = {
         pImprovement?: number;
         /** Number of paired item differences used for CI computation. */
         n?: number;
+        /**
+         * Per-side scorer success/failure counts over the SAME truncated pool the
+         * means and CI are computed from (not a separately-recomputed raw set), so
+         * a gate reads numbers consistent with what the table displays. Raw counts
+         * (consumer divides) keep this truncation-consistent. A non-zero
+         * `*Failed` means that side's mean rests on a thinned sample — the gate-side
+         * `--max-scorer-error-rate` flag refuses to certify when it's over tolerance.
+         */
+        baselineScored?: number;
+        baselineFailed?: number;
+        candidateScored?: number;
+        candidateFailed?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -297,6 +421,48 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 }>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
+/**
+ * Decide whether a comparison should be REFUSED under a gate-side scorer
+ * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and
+ * testable — returns a human-readable refusal reason, or `null` to allow.
+ *
+ * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY
+ * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a
+ * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`
+ * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the
+ * permissive tier) is deliberate: classifying an old artifact's scorers as
+ * deterministic would hard-fail them on the first flake, contradicting the
+ * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical
+ * across sides (enforced by `evalCompare`), so reading types from the baseline
+ * is sufficient.
+ *
+ * Also refuses a scorer that produced ZERO scored items on a side — a mean over
+ * an empty sample can't be certified. Counts come from the same truncated pool
+ * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).
+ */
+declare function evaluateScorerErrorRateGate(comparison: EvalComparison, maxScorerErrorRate: number): string | null;
+
+/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */
+type ScorerToleranceVerdict = {
+    /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */
+    attempted: number;
+    /** `failed / attempted`, or `0` when nothing was attempted. */
+    rate: number;
+    /** Over tolerance AND something was actually attempted. */
+    exceeds: boolean;
+    /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */
+    zeroSample: boolean;
+};
+/**
+ * Single source of truth for the type-aware failure-rate decision, shared by the
+ * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare
+ * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures
+ * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`
+ * (`failed / attempted`). `exceeds` is never true when nothing was attempted —
+ * callers decide their own policy for `zeroSample` (the runner skips it; the
+ * compare gate refuses to certify a zero-sample scorer).
+ */
+declare function evaluateScorerTolerance(scored: number, failed: number, type: 'llm' | 'deterministic', limit: number): ScorerToleranceVerdict;
 
 type BootstrapCIResult = {
     /** Lower bound of the confidence interval. */
@@ -323,7 +489,12 @@ declare function pairedBootstrapCI(differences: number[], options?: {
 }): BootstrapCIResult;
 
 type RescoreOptions = {
+    /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */
     concurrency?: number;
+    /** Per-item scorer fan-out (how many scorers run concurrently within one item).
+     *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent
+     *  judge calls is `concurrency × scorerConcurrency`. */
+    scorerConcurrency?: number;
     /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can
      *  be cancelled mid-flight. Also checked between items to short-circuit
      *  remaining work. Mirrors `RunEvalOptions.signal`. */
@@ -353,6 +524,10 @@ type MultiRunSummary = {
         std: number;
         min: number;
         max: number;
+        /** Total valid scores summed across all runs in the group. */
+        scored?: number;
+        /** Total ran-and-failed scorer invocations summed across all runs. */
+        failed?: number;
     }>;
     timing?: {
         mean: number;
@@ -364,4 +539,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.js

@@ -1,21 +1,84 @@
 import {
   aggregateRuns,
+  attachScorerErrorCost,
   evalCompare,
+  evaluateScorerErrorRateGate,
+  evaluateScorerTolerance,
   normalizeScorerResult,
   pairedBootstrapCI,
   rescore,
   runEval,
   scorer
-} from "./chunk-NETD4AUB.js";
+} from "./chunk-BWBX5YVQ.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
 import * as path from "path";
+function isPlainObject(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+function isObjectSchema(schema) {
+  const shape = schema.shape;
+  return typeof shape === "object" && shape !== null;
+}
+function collectDroppedKeys(raw, parsed, prefix = "") {
+  if (Array.isArray(raw) && Array.isArray(parsed)) {
+    const dropped = [];
+    const n = Math.min(raw.length, parsed.length);
+    for (let i = 0; i < n; i++) {
+      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));
+    }
+    return dropped;
+  }
+  if (isPlainObject(raw) && isPlainObject(parsed)) {
+    const dropped = [];
+    for (const key of Object.keys(raw)) {
+      const path2 = prefix ? `${prefix}.${key}` : key;
+      if (!(key in parsed)) {
+        dropped.push(path2);
+      } else {
+        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path2));
+      }
+    }
+    return dropped;
+  }
+  return [];
+}
 function dataset(config) {
+  const policy = config.onExtraAnnotationKeys ?? "warn";
+  function parseItem(item, droppedAcc) {
+    const input = config.schema.parse(item.input);
+    if (item.annotations == null || !config.annotations) {
+      return { input, annotations: item.annotations };
+    }
+    const annotations = config.annotations.parse(item.annotations);
+    if (policy !== "ignore" && isObjectSchema(config.annotations)) {
+      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);
+    }
+    return { input, annotations };
+  }
+  function reportDropped(dropped) {
+    if (dropped.size === 0) return;
+    const keys = [...dropped].join(", ");
+    if (policy === "error") {
+      throw new Error(
+        `Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will never reach scorers. Add them to the schema, or set onExtraAnnotationKeys: 'ignore' to allow stripping.`
+      );
+    }
+    console.warn(
+      `[axl-eval] Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will not reach scorers (a common cause of no-op scorers). Add them to the schema, set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`
+    );
+  }
+  let lastDroppedAnnotationKeys = [];
   return {
     name: config.name,
     schema: config.schema,
     annotationsSchema: config.annotations,
+    get droppedAnnotationKeys() {
+      return [...lastDroppedAnnotationKeys];
+    },
     async getItems() {
       if (config.items && config.file) {
         throw new Error('Dataset config error: "items" and "file" are mutually exclusive');
@@ -23,19 +86,19 @@ function dataset(config) {
       if (!config.items && !config.file) {
         throw new Error('Dataset config error: either "items" or "file" must be provided');
       }
+      let rawItems;
       if (config.items) {
-        return config.items.map((item) => ({
-          input: config.schema.parse(item.input),
-          annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-        }));
+        rawItems = config.items;
+      } else {
+        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
+        const content = await readFile(resolvedPath, "utf-8");
+        rawItems = JSON.parse(content);
       }
-      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
-      const content = await readFile(resolvedPath, "utf-8");
-      const items = JSON.parse(content);
-      return items.map((item) => ({
-        input: config.schema.parse(item.input),
-        annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-      }));
+      const dropped = /* @__PURE__ */ new Set();
+      const parsed = rawItems.map((item) => parseItem(item, dropped));
+      lastDroppedAnnotationKeys = [...dropped];
+      reportDropped(dropped);
+      return parsed;
     }
   };
 }
@@ -96,16 +159,14 @@ ${JSON.stringify(annotations, null, 2)}` : "",
         const parsed = JSON.parse(extractJson(response.content));
         validated = schema.parse(parsed);
       } catch (err) {
-        if (err && typeof err === "object") {
-          err.cost = responseCost;
-        }
+        attachScorerErrorCost(err, responseCost);
         if (err && typeof err === "object" && "issues" in err && Array.isArray(err.issues)) {
           const issues = err.issues;
           const messages = issues.map((i) => `${i.path.length ? `${i.path.join(".")}: ` : ""}${i.message}`).join("; ");
           const error = new Error(
             `LLM scorer "${config.name}" returned an invalid response: ${messages}`
           );
-          error.cost = responseCost;
+          attachScorerErrorCost(error, responseCost);
           throw error;
         }
         throw err;
@@ -129,6 +190,8 @@ export {
   dataset,
   defineEval,
   evalCompare,
+  evaluateScorerErrorRateGate,
+  evaluateScorerTolerance,
   llmScorer,
   normalizeScorerResult,
   pairedBootstrapCI,