@axlsdk/eval 0.17.9 → 0.18.1

package/dist/index.d.ts

@@ -5,6 +5,27 @@ type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
     annotations?: TAnnotations;
 };
+/**
+ * What to do when a dataset item's `annotations` contain keys that are absent
+ * from the `annotations` schema.
+ *
+ * Zod object schemas strip unknown keys by default, so an annotation field
+ * that isn't declared in the schema is silently dropped before it ever reaches
+ * a scorer. A scorer that reads such a field sees `undefined` and quietly
+ * degrades to a no-op — the eval still "passes", so the bug is invisible. This
+ * is especially easy to hit with file-based datasets, where the JSON is never
+ * type-checked.
+ *
+ * - `'warn'` (default) — log a single `console.warn` per dataset listing the
+ *   dropped key paths. Non-breaking; surfaces the contract violation.
+ * - `'error'` — throw, listing the dropped key paths. The strict path for CI.
+ * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).
+ *
+ * Only `annotations` are checked, not `input`: stripped input fields flow to
+ * the workflow where they surface, whereas stripped annotations vanish into a
+ * silent scorer.
+ */
+type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';
 type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {
     name: string;
     schema: TInput;
@@ -13,11 +34,28 @@ type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.
     file?: string;
     /** Base directory for resolving relative file paths. Defaults to cwd. */
     basePath?: string;
+    /**
+     * How to handle annotation keys not present in the `annotations` schema
+     * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.
+     */
+    onExtraAnnotationKeys?: ExtraKeyPolicy;
 };
 type Dataset<TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly schema: z.ZodType;
     readonly annotationsSchema?: z.ZodType;
+    /**
+     * Annotation key paths the schema dropped during the most recent
+     * `getItems()` call (empty when none were dropped, or when
+     * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of
+     * `getItems()` — read it synchronously right after awaiting, on the same
+     * instance; a later `getItems()` overwrites it (calling `getItems()`
+     * concurrently on one shared dataset is unsupported). The eval runner folds
+     * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the
+     * CLI logs it; UIs can render it) sees the same signal. Optional so
+     * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.
+     */
+    readonly droppedAnnotationKeys?: readonly string[];
     getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
@@ -41,16 +79,51 @@ type ScorerContext = {
     signal?: AbortSignal;
 };
 type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
+/**
+ * Applicability predicate for a conditional scorer. Returns `true` if this
+ * scorer should run for the given item, `false` to skip it.
+ *
+ * The eval runner runs every scorer against every item, so a scorer that only
+ * applies to a subset (a refusal judge for refusal-expected items, a constraint
+ * judge for constrained items, …) declares its scope here. A skipped item is
+ * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND
+ * from the failure-rate denominator, so a conditional scorer stays honest to
+ * both. The predicate runs BEFORE the scorer body, so an inapplicable
+ * `llmScorer` never makes its provider call.
+ *
+ * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability
+ * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but
+ * `output` is available too (e.g. "only score outputs that refused"). A
+ * predicate that THROWS is a bug, not a skip — it's recorded as a scorer
+ * failure, never silently swallowed.
+ *
+ * The contract is `=> boolean`: return `true` to run, `false` to skip. The
+ * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy
+ * value (a predicate that forgot to `return`) skips rather than crashes — but
+ * authors should return a real boolean. When deriving from a truthy value,
+ * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not
+ * `=> a.constraints?.length` (which is `number | undefined`, a type error). A
+ * predicate that accidentally never returns therefore skips EVERY item — but
+ * that surfaces conspicuously as a non-zero `skipped` count (the Studio "N/A"
+ * chip), not as a silently green mean.
+ */
+type ScorerApplies<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => boolean;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
     score: ScorerFn<TOutput, TInput, TAnnotations>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the scorer is skipped for that item (counted as neither
+     *  scored nor failed). Omit to run on every item (the default). */
+    applies?: ScorerApplies<TOutput, TInput, TAnnotations>;
 };
 type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
     score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
+    /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}. */
+    applies?: ScorerApplies<TOutput, TInput, TAnnotations>;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -79,6 +152,11 @@ type LlmScorerConfig = {
     stop?: string[];
     /** Provider-specific escape hatch — merged last into the raw API request. */
     providerOptions?: Record<string, unknown>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the judge is skipped for that item and NO provider call is
+     *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as
+     *  neither scored nor failed. */
+    applies?: ScorerApplies<unknown, unknown, unknown>;
 };
 declare function llmScorer(config: LlmScorerConfig): Scorer;
 
@@ -86,10 +164,53 @@ type EvalConfig = {
     workflow: string;
     dataset: Dataset<unknown, unknown>;
     scorers: Scorer<unknown, unknown, unknown>[];
+    /** Item-level worker-pool size (how many dataset items run in parallel). Default 5. */
     concurrency?: number;
+    /**
+     * Per-item scorer fan-out — how many of an item's scorers run concurrently.
+     * Default 5 (matches `concurrency`; scorers are parallel-by-default). The
+     * worst-case number of concurrent scorer calls is `concurrency × scorerConcurrency`,
+     * so lower `concurrency` if a rate-limited judge model needs a tighter ceiling.
+     */
+    scorerConcurrency?: number;
     budget?: string;
+    /**
+     * Opt-in source-side trust gate. When set (0–1), `runEval` flags the run as
+     * `summary.degraded` if any scorer's failure rate exceeds tolerance, and the
+     * CLI exits non-zero. Enforcement is **type-aware**: deterministic scorers
+     * tolerate ZERO failures (a deterministic scorer that throws is a bug, not
+     * noise), while LLM scorers use this rate (a flaky judge / rate-limit storm).
+     *
+     * Failure rate = `failed / (scored + failed)` per scorer, where `scored` is
+     * the number of items that produced a valid numeric score and `failed` is the
+     * number whose scorer ran and threw / returned out-of-range. `0` means "any
+     * LLM failure degrades the run". Invalid values (non-finite, <0, >1) are
+     * ignored with a `console.warn`.
+     *
+     * Distinct from the gate-side `--max-scorer-error-rate` compare flag: this one
+     * fires at run/produce time (catch a thinned sample before it's saved), the
+     * other at consume/gate time (refuse to certify a thinned baseline/candidate).
+     */
+    failOnScorerErrorRate?: number;
     metadata?: Record<string, unknown>;
 };
+/**
+ * One scorer whose failure rate tripped `EvalConfig.failOnScorerErrorRate`.
+ * Surfaced on `EvalSummary.degraded` so consumers (CLI exit code, Studio
+ * banner) can refuse to trust the mean without re-deriving the rate.
+ */
+type DegradedScorer = {
+    scorer: string;
+    /** Observed failure rate `failed / (scored + failed)` (0 when nothing ran). */
+    rate: number;
+    /** The tolerance that was exceeded (0 for deterministic scorers). */
+    limit: number;
+    type: 'llm' | 'deterministic';
+    /** Items that produced a valid numeric score. */
+    scored: number;
+    /** Items whose scorer ran and failed (threw / out-of-range). */
+    failed: number;
+};
 type EvalResult = {
     id: string;
     /**
@@ -108,6 +229,13 @@ type EvalResult = {
      *   that need a single "primary" workflow should use `workflows[0]`.
      * - `scorerTypes: Record<string, 'llm' | 'deterministic'>`
      * - `runGroupId?: string`, `runIndex?: number` for multi-run groups
+     * - `droppedAnnotationKeys?: string[]` — annotation key paths the dataset
+     *   schema stripped (present only when non-empty). Mirrors the dataset's
+     *   `console.warn`; lets any consumer surface the same signal.
+     * - `scorerFiltered?: boolean` + `scorersRun?: string[]` — set when the CLI
+     *   `--scorers` flag ran a subset of scorers. A guard so a filtered run isn't
+     *   silently used as a full baseline (`compare` warns / refuses to gate on it).
+     *   Distinct from the multi-run "partial" concept (see {@link EvalComparisonPartial}).
      */
     metadata: Record<string, unknown>;
     timestamp: string;
@@ -121,6 +249,14 @@ type ScorerDetail = {
     metadata?: Record<string, unknown>;
     duration?: number;
     cost?: number;
+    /**
+     * `true` when the scorer's `applies` predicate returned `false` for this item,
+     * so the scorer was deliberately skipped (NOT run). Distinct from a `null`
+     * score WITH a `duration` (ran-and-failed) and from a `null` score with
+     * neither field (skipped by cancellation). A skipped item is excluded from the
+     * mean AND from the failure-rate denominator — see `scorerCounts`.
+     */
+    skipped?: boolean;
 };
 type EvalItem = {
     input: unknown;
@@ -150,6 +286,29 @@ type EvalSummary = {
         max: number;
         p50: number;
         p95: number;
+        /**
+         * Number of items that produced a valid numeric score — the sample size
+         * the `mean` actually covers. Optional so pre-existing artifacts (and
+         * hand-rolled summaries) stay valid; absent ⇒ recompute from `items`.
+         * Named `scored` (NOT `n`) to avoid collision with
+         * `EvalComparison.scorers[].n` (paired-diff count).
+         */
+        scored?: number;
+        /**
+         * Number of items whose scorer RAN and failed (threw or returned
+         * out-of-range). Distinct from items skipped by cancellation, which land
+         * in neither bucket — so `scored + failed` is the honest "attempted"
+         * denominator and may be `<` the eligible item count. A non-zero `failed`
+         * means the `mean` was computed over a thinned sample.
+         */
+        failed?: number;
+        /**
+         * Number of items for which this scorer's `applies` predicate returned
+         * `false` — deliberately skipped, NOT run. Counted in neither `scored` nor
+         * `failed`, so it never inflates the failure rate. Optional so pre-existing
+         * artifacts stay valid; absent ⇒ recompute from `items` (0 if no skips).
+         */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -158,6 +317,14 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /**
+     * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
+     * and one or more scorers exceeded tolerance. `runEval` never throws on this
+     * — it returns the (still-useful) result with this flag set, and the CLI /
+     * consumer decides whether to fail. Absent / empty ⇒ no degradation gate
+     * tripped (either not configured or all scorers within tolerance).
+     */
+    degraded?: DegradedScorer[];
 };
 /**
  * Sample-size context for one side of a comparison.
@@ -214,8 +381,39 @@ type EvalComparison = {
         significant?: boolean;
         pRegression?: number;
         pImprovement?: number;
-        /** Number of paired item differences used for CI computation. */
+        /**
+         * Paired sample size — the count of items scored on BOTH sides (the only
+         * items a difference exists for). Always set, including `0`/`1`. The CI /
+         * `significant` / `pRegression` / `pImprovement` fields are populated only
+         * when `n >= 2`. NOTE the asymmetry: `delta` is the difference of the two
+         * INDEPENDENT per-side means (each over `{baseline,candidate}Scored`
+         * items), whereas the CI is paired over these `n` — so when `n` is much
+         * smaller than the per-side scored counts (different skips/failures per
+         * side), the delta and the CI rest on different samples.
+         */
         n?: number;
+        /**
+         * Per-side scorer success/failure counts over the SAME truncated pool the
+         * means and CI are computed from (not a separately-recomputed raw set), so
+         * a gate reads numbers consistent with what the table displays. Raw counts
+         * (consumer divides) keep this truncation-consistent. A non-zero
+         * `*Failed` means that side's mean rests on a thinned sample — the gate-side
+         * `--max-scorer-error-rate` flag refuses to certify when it's over tolerance.
+         */
+        baselineScored?: number;
+        baselineFailed?: number;
+        candidateScored?: number;
+        candidateFailed?: number;
+        /**
+         * Per-side `applies`-skipped (N/A) counts over the same truncated pool.
+         * Skips are excluded from both the mean and the failure-rate denominator,
+         * so they never move the delta — but a side that skipped items scored a
+         * different subset than a side that didn't, so surfacing the count lets a
+         * consumer explain a delta that's really a sample mismatch rather than a
+         * regression.
+         */
+        baselineSkipped?: number;
+        candidateSkipped?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -297,6 +495,48 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 }>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
+/**
+ * Decide whether a comparison should be REFUSED under a gate-side scorer
+ * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and
+ * testable — returns a human-readable refusal reason, or `null` to allow.
+ *
+ * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY
+ * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a
+ * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`
+ * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the
+ * permissive tier) is deliberate: classifying an old artifact's scorers as
+ * deterministic would hard-fail them on the first flake, contradicting the
+ * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical
+ * across sides (enforced by `evalCompare`), so reading types from the baseline
+ * is sufficient.
+ *
+ * Also refuses a scorer that produced ZERO scored items on a side — a mean over
+ * an empty sample can't be certified. Counts come from the same truncated pool
+ * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).
+ */
+declare function evaluateScorerErrorRateGate(comparison: EvalComparison, maxScorerErrorRate: number): string | null;
+
+/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */
+type ScorerToleranceVerdict = {
+    /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */
+    attempted: number;
+    /** `failed / attempted`, or `0` when nothing was attempted. */
+    rate: number;
+    /** Over tolerance AND something was actually attempted. */
+    exceeds: boolean;
+    /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */
+    zeroSample: boolean;
+};
+/**
+ * Single source of truth for the type-aware failure-rate decision, shared by the
+ * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare
+ * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures
+ * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`
+ * (`failed / attempted`). `exceeds` is never true when nothing was attempted —
+ * callers decide their own policy for `zeroSample` (the runner skips it; the
+ * compare gate refuses to certify a zero-sample scorer).
+ */
+declare function evaluateScorerTolerance(scored: number, failed: number, type: 'llm' | 'deterministic', limit: number): ScorerToleranceVerdict;
 
 type BootstrapCIResult = {
     /** Lower bound of the confidence interval. */
@@ -323,7 +563,12 @@ declare function pairedBootstrapCI(differences: number[], options?: {
 }): BootstrapCIResult;
 
 type RescoreOptions = {
+    /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */
     concurrency?: number;
+    /** Per-item scorer fan-out (how many scorers run concurrently within one item).
+     *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent
+     *  judge calls is `concurrency × scorerConcurrency`. */
+    scorerConcurrency?: number;
     /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can
      *  be cancelled mid-flight. Also checked between items to short-circuit
      *  remaining work. Mirrors `RunEvalOptions.signal`. */
@@ -353,6 +598,13 @@ type MultiRunSummary = {
         std: number;
         min: number;
         max: number;
+        /** Total valid scores summed across all runs in the group. */
+        scored?: number;
+        /** Total ran-and-failed scorer invocations summed across all runs. */
+        failed?: number;
+        /** Total `applies`-skipped (N/A) items summed across all runs. Excluded
+         *  from both the mean and the failure-rate denominator. */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -364,4 +616,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.js

@@ -1,21 +1,84 @@
 import {
   aggregateRuns,
+  attachScorerErrorCost,
   evalCompare,
+  evaluateScorerErrorRateGate,
+  evaluateScorerTolerance,
   normalizeScorerResult,
   pairedBootstrapCI,
   rescore,
   runEval,
   scorer
-} from "./chunk-NETD4AUB.js";
+} from "./chunk-66EYSYQS.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
 import * as path from "path";
+function isPlainObject(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+function isObjectSchema(schema) {
+  const shape = schema.shape;
+  return typeof shape === "object" && shape !== null;
+}
+function collectDroppedKeys(raw, parsed, prefix = "") {
+  if (Array.isArray(raw) && Array.isArray(parsed)) {
+    const dropped = [];
+    const n = Math.min(raw.length, parsed.length);
+    for (let i = 0; i < n; i++) {
+      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));
+    }
+    return dropped;
+  }
+  if (isPlainObject(raw) && isPlainObject(parsed)) {
+    const dropped = [];
+    for (const key of Object.keys(raw)) {
+      const path2 = prefix ? `${prefix}.${key}` : key;
+      if (!(key in parsed)) {
+        dropped.push(path2);
+      } else {
+        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path2));
+      }
+    }
+    return dropped;
+  }
+  return [];
+}
 function dataset(config) {
+  const policy = config.onExtraAnnotationKeys ?? "warn";
+  function parseItem(item, droppedAcc) {
+    const input = config.schema.parse(item.input);
+    if (item.annotations == null || !config.annotations) {
+      return { input, annotations: item.annotations };
+    }
+    const annotations = config.annotations.parse(item.annotations);
+    if (policy !== "ignore" && isObjectSchema(config.annotations)) {
+      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);
+    }
+    return { input, annotations };
+  }
+  function reportDropped(dropped) {
+    if (dropped.size === 0) return;
+    const keys = [...dropped].join(", ");
+    if (policy === "error") {
+      throw new Error(
+        `Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will never reach scorers. Add them to the schema, or set onExtraAnnotationKeys: 'ignore' to allow stripping.`
+      );
+    }
+    console.warn(
+      `[axl-eval] Dataset "${config.name}": annotation key(s) dropped by schema: ${keys}. These keys are not declared in the annotations schema, so they were stripped and will not reach scorers (a common cause of no-op scorers). Add them to the schema, set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`
+    );
+  }
+  let lastDroppedAnnotationKeys = [];
   return {
     name: config.name,
     schema: config.schema,
     annotationsSchema: config.annotations,
+    get droppedAnnotationKeys() {
+      return [...lastDroppedAnnotationKeys];
+    },
     async getItems() {
       if (config.items && config.file) {
         throw new Error('Dataset config error: "items" and "file" are mutually exclusive');
@@ -23,19 +86,19 @@ function dataset(config) {
       if (!config.items && !config.file) {
         throw new Error('Dataset config error: either "items" or "file" must be provided');
       }
+      let rawItems;
       if (config.items) {
-        return config.items.map((item) => ({
-          input: config.schema.parse(item.input),
-          annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-        }));
+        rawItems = config.items;
+      } else {
+        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
+        const content = await readFile(resolvedPath, "utf-8");
+        rawItems = JSON.parse(content);
       }
-      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
-      const content = await readFile(resolvedPath, "utf-8");
-      const items = JSON.parse(content);
-      return items.map((item) => ({
-        input: config.schema.parse(item.input),
-        annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
-      }));
+      const dropped = /* @__PURE__ */ new Set();
+      const parsed = rawItems.map((item) => parseItem(item, dropped));
+      lastDroppedAnnotationKeys = [...dropped];
+      reportDropped(dropped);
+      return parsed;
     }
   };
 }
@@ -50,6 +113,7 @@ function llmScorer(config) {
     name: config.name,
     description: config.description,
     isLlm: true,
+    applies: config.applies,
     async score(output, input, annotations, context) {
       if (!context?.resolveProvider) {
         throw new Error(
@@ -96,16 +160,14 @@ ${JSON.stringify(annotations, null, 2)}` : "",
         const parsed = JSON.parse(extractJson(response.content));
         validated = schema.parse(parsed);
       } catch (err) {
-        if (err && typeof err === "object") {
-          err.cost = responseCost;
-        }
+        attachScorerErrorCost(err, responseCost);
         if (err && typeof err === "object" && "issues" in err && Array.isArray(err.issues)) {
           const issues = err.issues;
           const messages = issues.map((i) => `${i.path.length ? `${i.path.join(".")}: ` : ""}${i.message}`).join("; ");
           const error = new Error(
             `LLM scorer "${config.name}" returned an invalid response: ${messages}`
           );
-          error.cost = responseCost;
+          attachScorerErrorCost(error, responseCost);
           throw error;
         }
         throw err;
@@ -129,6 +191,8 @@ export {
   dataset,
   defineEval,
   evalCompare,
+  evaluateScorerErrorRateGate,
+  evaluateScorerTolerance,
   llmScorer,
   normalizeScorerResult,
   pairedBootstrapCI,

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        if (err && typeof err === 'object') {\n          (err as any).cost = responseCost;\n        }\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          (error as any).cost = responseCost;\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7DA,SAAS,SAAS;AAElB,SAAS,aAAa,uBAAuB;AA2BtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;AC1IO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AAgCtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;AC/IO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.17.9",
+  "version": "0.18.1",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.17.9"
+    "@axlsdk/axl": "0.18.1"
   },
   "peerDependencies": {
     "zod": "^4.0.0",
@@ -71,6 +71,7 @@
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "test:watch": "vitest watch",
+    "test:integration": "vitest run --config vitest.integration.config.ts",
     "lint": "tsc --noEmit"
   }
 }