@axlsdk/eval 0.17.9 → 0.18.1

package/dist/index.d.cts

@@ -5,6 +5,27 @@ type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
     annotations?: TAnnotations;
 };
+/**
+ * What to do when a dataset item's `annotations` contain keys that are absent
+ * from the `annotations` schema.
+ *
+ * Zod object schemas strip unknown keys by default, so an annotation field
+ * that isn't declared in the schema is silently dropped before it ever reaches
+ * a scorer. A scorer that reads such a field sees `undefined` and quietly
+ * degrades to a no-op — the eval still "passes", so the bug is invisible. This
+ * is especially easy to hit with file-based datasets, where the JSON is never
+ * type-checked.
+ *
+ * - `'warn'` (default) — log a single `console.warn` per dataset listing the
+ *   dropped key paths. Non-breaking; surfaces the contract violation.
+ * - `'error'` — throw, listing the dropped key paths. The strict path for CI.
+ * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).
+ *
+ * Only `annotations` are checked, not `input`: stripped input fields flow to
+ * the workflow where they surface, whereas stripped annotations vanish into a
+ * silent scorer.
+ */
+type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';
 type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {
     name: string;
     schema: TInput;
@@ -13,11 +34,28 @@ type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.
     file?: string;
     /** Base directory for resolving relative file paths. Defaults to cwd. */
     basePath?: string;
+    /**
+     * How to handle annotation keys not present in the `annotations` schema
+     * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.
+     */
+    onExtraAnnotationKeys?: ExtraKeyPolicy;
 };
 type Dataset<TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly schema: z.ZodType;
     readonly annotationsSchema?: z.ZodType;
+    /**
+     * Annotation key paths the schema dropped during the most recent
+     * `getItems()` call (empty when none were dropped, or when
+     * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of
+     * `getItems()` — read it synchronously right after awaiting, on the same
+     * instance; a later `getItems()` overwrites it (calling `getItems()`
+     * concurrently on one shared dataset is unsupported). The eval runner folds
+     * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the
+     * CLI logs it; UIs can render it) sees the same signal. Optional so
+     * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.
+     */
+    readonly droppedAnnotationKeys?: readonly string[];
     getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
@@ -41,16 +79,51 @@ type ScorerContext = {
     signal?: AbortSignal;
 };
 type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
+/**
+ * Applicability predicate for a conditional scorer. Returns `true` if this
+ * scorer should run for the given item, `false` to skip it.
+ *
+ * The eval runner runs every scorer against every item, so a scorer that only
+ * applies to a subset (a refusal judge for refusal-expected items, a constraint
+ * judge for constrained items, …) declares its scope here. A skipped item is
+ * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND
+ * from the failure-rate denominator, so a conditional scorer stays honest to
+ * both. The predicate runs BEFORE the scorer body, so an inapplicable
+ * `llmScorer` never makes its provider call.
+ *
+ * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability
+ * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but
+ * `output` is available too (e.g. "only score outputs that refused"). A
+ * predicate that THROWS is a bug, not a skip — it's recorded as a scorer
+ * failure, never silently swallowed.
+ *
+ * The contract is `=> boolean`: return `true` to run, `false` to skip. The
+ * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy
+ * value (a predicate that forgot to `return`) skips rather than crashes — but
+ * authors should return a real boolean. When deriving from a truthy value,
+ * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not
+ * `=> a.constraints?.length` (which is `number | undefined`, a type error). A
+ * predicate that accidentally never returns therefore skips EVERY item — but
+ * that surfaces conspicuously as a non-zero `skipped` count (the Studio "N/A"
+ * chip), not as a silently green mean.
+ */
+type ScorerApplies<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => boolean;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
     score: ScorerFn<TOutput, TInput, TAnnotations>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the scorer is skipped for that item (counted as neither
+     *  scored nor failed). Omit to run on every item (the default). */
+    applies?: ScorerApplies<TOutput, TInput, TAnnotations>;
 };
 type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
     score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
+    /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}. */
+    applies?: ScorerApplies<TOutput, TInput, TAnnotations>;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -79,6 +152,11 @@ type LlmScorerConfig = {
     stop?: string[];
     /** Provider-specific escape hatch — merged last into the raw API request. */
     providerOptions?: Record<string, unknown>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the judge is skipped for that item and NO provider call is
+     *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as
+     *  neither scored nor failed. */
+    applies?: ScorerApplies<unknown, unknown, unknown>;
 };
 declare function llmScorer(config: LlmScorerConfig): Scorer;
 
@@ -86,10 +164,53 @@ type EvalConfig = {
     workflow: string;
     dataset: Dataset<unknown, unknown>;
     scorers: Scorer<unknown, unknown, unknown>[];
+    /** Item-level worker-pool size (how many dataset items run in parallel). Default 5. */
     concurrency?: number;
+    /**
+     * Per-item scorer fan-out — how many of an item's scorers run concurrently.
+     * Default 5 (matches `concurrency`; scorers are parallel-by-default). The
+     * worst-case number of concurrent scorer calls is `concurrency × scorerConcurrency`,
+     * so lower `concurrency` if a rate-limited judge model needs a tighter ceiling.
+     */
+    scorerConcurrency?: number;
     budget?: string;
+    /**
+     * Opt-in source-side trust gate. When set (0–1), `runEval` flags the run as
+     * `summary.degraded` if any scorer's failure rate exceeds tolerance, and the
+     * CLI exits non-zero. Enforcement is **type-aware**: deterministic scorers
+     * tolerate ZERO failures (a deterministic scorer that throws is a bug, not
+     * noise), while LLM scorers use this rate (a flaky judge / rate-limit storm).
+     *
+     * Failure rate = `failed / (scored + failed)` per scorer, where `scored` is
+     * the number of items that produced a valid numeric score and `failed` is the
+     * number whose scorer ran and threw / returned out-of-range. `0` means "any
+     * LLM failure degrades the run". Invalid values (non-finite, <0, >1) are
+     * ignored with a `console.warn`.
+     *
+     * Distinct from the gate-side `--max-scorer-error-rate` compare flag: this one
+     * fires at run/produce time (catch a thinned sample before it's saved), the
+     * other at consume/gate time (refuse to certify a thinned baseline/candidate).
+     */
+    failOnScorerErrorRate?: number;
     metadata?: Record<string, unknown>;
 };
+/**
+ * One scorer whose failure rate tripped `EvalConfig.failOnScorerErrorRate`.
+ * Surfaced on `EvalSummary.degraded` so consumers (CLI exit code, Studio
+ * banner) can refuse to trust the mean without re-deriving the rate.
+ */
+type DegradedScorer = {
+    scorer: string;
+    /** Observed failure rate `failed / (scored + failed)` (0 when nothing ran). */
+    rate: number;
+    /** The tolerance that was exceeded (0 for deterministic scorers). */
+    limit: number;
+    type: 'llm' | 'deterministic';
+    /** Items that produced a valid numeric score. */
+    scored: number;
+    /** Items whose scorer ran and failed (threw / out-of-range). */
+    failed: number;
+};
 type EvalResult = {
     id: string;
     /**
@@ -108,6 +229,13 @@ type EvalResult = {
      *   that need a single "primary" workflow should use `workflows[0]`.
      * - `scorerTypes: Record<string, 'llm' | 'deterministic'>`
      * - `runGroupId?: string`, `runIndex?: number` for multi-run groups
+     * - `droppedAnnotationKeys?: string[]` — annotation key paths the dataset
+     *   schema stripped (present only when non-empty). Mirrors the dataset's
+     *   `console.warn`; lets any consumer surface the same signal.
+     * - `scorerFiltered?: boolean` + `scorersRun?: string[]` — set when the CLI
+     *   `--scorers` flag ran a subset of scorers. A guard so a filtered run isn't
+     *   silently used as a full baseline (`compare` warns / refuses to gate on it).
+     *   Distinct from the multi-run "partial" concept (see {@link EvalComparisonPartial}).
      */
     metadata: Record<string, unknown>;
     timestamp: string;
@@ -121,6 +249,14 @@ type ScorerDetail = {
     metadata?: Record<string, unknown>;
     duration?: number;
     cost?: number;
+    /**
+     * `true` when the scorer's `applies` predicate returned `false` for this item,
+     * so the scorer was deliberately skipped (NOT run). Distinct from a `null`
+     * score WITH a `duration` (ran-and-failed) and from a `null` score with
+     * neither field (skipped by cancellation). A skipped item is excluded from the
+     * mean AND from the failure-rate denominator — see `scorerCounts`.
+     */
+    skipped?: boolean;
 };
 type EvalItem = {
     input: unknown;
@@ -150,6 +286,29 @@ type EvalSummary = {
         max: number;
         p50: number;
         p95: number;
+        /**
+         * Number of items that produced a valid numeric score — the sample size
+         * the `mean` actually covers. Optional so pre-existing artifacts (and
+         * hand-rolled summaries) stay valid; absent ⇒ recompute from `items`.
+         * Named `scored` (NOT `n`) to avoid collision with
+         * `EvalComparison.scorers[].n` (paired-diff count).
+         */
+        scored?: number;
+        /**
+         * Number of items whose scorer RAN and failed (threw or returned
+         * out-of-range). Distinct from items skipped by cancellation, which land
+         * in neither bucket — so `scored + failed` is the honest "attempted"
+         * denominator and may be `<` the eligible item count. A non-zero `failed`
+         * means the `mean` was computed over a thinned sample.
+         */
+        failed?: number;
+        /**
+         * Number of items for which this scorer's `applies` predicate returned
+         * `false` — deliberately skipped, NOT run. Counted in neither `scored` nor
+         * `failed`, so it never inflates the failure rate. Optional so pre-existing
+         * artifacts stay valid; absent ⇒ recompute from `items` (0 if no skips).
+         */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -158,6 +317,14 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /**
+     * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
+     * and one or more scorers exceeded tolerance. `runEval` never throws on this
+     * — it returns the (still-useful) result with this flag set, and the CLI /
+     * consumer decides whether to fail. Absent / empty ⇒ no degradation gate
+     * tripped (either not configured or all scorers within tolerance).
+     */
+    degraded?: DegradedScorer[];
 };
 /**
  * Sample-size context for one side of a comparison.
@@ -214,8 +381,39 @@ type EvalComparison = {
         significant?: boolean;
         pRegression?: number;
         pImprovement?: number;
-        /** Number of paired item differences used for CI computation. */
+        /**
+         * Paired sample size — the count of items scored on BOTH sides (the only
+         * items a difference exists for). Always set, including `0`/`1`. The CI /
+         * `significant` / `pRegression` / `pImprovement` fields are populated only
+         * when `n >= 2`. NOTE the asymmetry: `delta` is the difference of the two
+         * INDEPENDENT per-side means (each over `{baseline,candidate}Scored`
+         * items), whereas the CI is paired over these `n` — so when `n` is much
+         * smaller than the per-side scored counts (different skips/failures per
+         * side), the delta and the CI rest on different samples.
+         */
         n?: number;
+        /**
+         * Per-side scorer success/failure counts over the SAME truncated pool the
+         * means and CI are computed from (not a separately-recomputed raw set), so
+         * a gate reads numbers consistent with what the table displays. Raw counts
+         * (consumer divides) keep this truncation-consistent. A non-zero
+         * `*Failed` means that side's mean rests on a thinned sample — the gate-side
+         * `--max-scorer-error-rate` flag refuses to certify when it's over tolerance.
+         */
+        baselineScored?: number;
+        baselineFailed?: number;
+        candidateScored?: number;
+        candidateFailed?: number;
+        /**
+         * Per-side `applies`-skipped (N/A) counts over the same truncated pool.
+         * Skips are excluded from both the mean and the failure-rate denominator,
+         * so they never move the delta — but a side that skipped items scored a
+         * different subset than a side that didn't, so surfacing the count lets a
+         * consumer explain a delta that's really a sample mismatch rather than a
+         * regression.
+         */
+        baselineSkipped?: number;
+        candidateSkipped?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -297,6 +495,48 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 }>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
+/**
+ * Decide whether a comparison should be REFUSED under a gate-side scorer
+ * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and
+ * testable — returns a human-readable refusal reason, or `null` to allow.
+ *
+ * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY
+ * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a
+ * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`
+ * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the
+ * permissive tier) is deliberate: classifying an old artifact's scorers as
+ * deterministic would hard-fail them on the first flake, contradicting the
+ * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical
+ * across sides (enforced by `evalCompare`), so reading types from the baseline
+ * is sufficient.
+ *
+ * Also refuses a scorer that produced ZERO scored items on a side — a mean over
+ * an empty sample can't be certified. Counts come from the same truncated pool
+ * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).
+ */
+declare function evaluateScorerErrorRateGate(comparison: EvalComparison, maxScorerErrorRate: number): string | null;
+
+/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */
+type ScorerToleranceVerdict = {
+    /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */
+    attempted: number;
+    /** `failed / attempted`, or `0` when nothing was attempted. */
+    rate: number;
+    /** Over tolerance AND something was actually attempted. */
+    exceeds: boolean;
+    /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */
+    zeroSample: boolean;
+};
+/**
+ * Single source of truth for the type-aware failure-rate decision, shared by the
+ * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare
+ * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures
+ * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`
+ * (`failed / attempted`). `exceeds` is never true when nothing was attempted —
+ * callers decide their own policy for `zeroSample` (the runner skips it; the
+ * compare gate refuses to certify a zero-sample scorer).
+ */
+declare function evaluateScorerTolerance(scored: number, failed: number, type: 'llm' | 'deterministic', limit: number): ScorerToleranceVerdict;
 
 type BootstrapCIResult = {
     /** Lower bound of the confidence interval. */
@@ -323,7 +563,12 @@ declare function pairedBootstrapCI(differences: number[], options?: {
 }): BootstrapCIResult;
 
 type RescoreOptions = {
+    /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */
     concurrency?: number;
+    /** Per-item scorer fan-out (how many scorers run concurrently within one item).
+     *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent
+     *  judge calls is `concurrency × scorerConcurrency`. */
+    scorerConcurrency?: number;
     /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can
      *  be cancelled mid-flight. Also checked between items to short-circuit
      *  remaining work. Mirrors `RunEvalOptions.signal`. */
@@ -353,6 +598,13 @@ type MultiRunSummary = {
         std: number;
         min: number;
         max: number;
+        /** Total valid scores summed across all runs in the group. */
+        scored?: number;
+        /** Total ran-and-failed scorer invocations summed across all runs. */
+        failed?: number;
+        /** Total `applies`-skipped (N/A) items summed across all runs. Excluded
+         *  from both the mean and the failure-rate denominator. */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -364,4 +616,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };