@axlsdk/eval 0.18.0 → 0.18.2

package/dist/index.d.cts

@@ -79,16 +79,64 @@ type ScorerContext = {
     signal?: AbortSignal;
 };
 type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
+/**
+ * Applicability predicate for a conditional scorer. Returns `true` if this
+ * scorer should run for the given item, `false` to skip it.
+ *
+ * The eval runner runs every scorer against every item, so a scorer that only
+ * applies to a subset (a refusal judge for refusal-expected items, a constraint
+ * judge for constrained items, …) declares its scope here. A skipped item is
+ * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND
+ * from the failure-rate denominator, so a conditional scorer stays honest to
+ * both. The predicate runs BEFORE the scorer body, so an inapplicable
+ * `llmScorer` never makes its provider call.
+ *
+ * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability
+ * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but
+ * `output` is available too (e.g. "only score outputs that refused"). A
+ * predicate that THROWS is a bug, not a skip — it's recorded as a scorer
+ * failure, never silently swallowed.
+ *
+ * The contract is `=> boolean`: return `true` to run, `false` to skip. The
+ * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy
+ * value (a predicate that forgot to `return`) skips rather than crashes — but
+ * authors should return a real boolean. When deriving from a truthy value,
+ * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not
+ * `=> a.constraints?.length` (which is `number | undefined`, a type error). A
+ * predicate that accidentally never returns therefore skips EVERY item — but
+ * that surfaces conspicuously as a non-zero `skipped` count (the Studio "N/A"
+ * chip), not as a silently green mean.
+ */
+type ScorerApplies<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => boolean;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
     score: ScorerFn<TOutput, TInput, TAnnotations>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the scorer is skipped for that item (counted as neither
+     *  scored nor failed). Omit to run on every item (the default).
+     *
+     *  Declared with **method syntax** (not a function-valued property) for parity
+     *  with {@link Scorer.applies}, where the syntax is load-bearing: see the note
+     *  there and `__tests__/scorer-assignability.test-d.ts`. */
+    applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;
 };
 type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
     score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
+    /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}.
+     *
+     *  Declared with **method syntax** (not a function-valued `ScorerApplies`
+     *  property) so its parameters are checked bivariantly, exactly like
+     *  {@link score}. `EvalConfig.scorers` erases the generics to
+     *  `Scorer<unknown, unknown, unknown>[]`; under `strictFunctionTypes` a
+     *  property would make those params contravariant, so a concretely-typed
+     *  `Scorer<…, TInput, TAnnotations>` would no longer be assignable to that
+     *  array (the 0.18.1 regression). `__tests__/scorer-assignability.test-d.ts`
+     *  locks this in and is compiled by the `typecheck` gate. */
+    applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -117,6 +165,11 @@ type LlmScorerConfig = {
     stop?: string[];
     /** Provider-specific escape hatch — merged last into the raw API request. */
     providerOptions?: Record<string, unknown>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the judge is skipped for that item and NO provider call is
+     *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as
+     *  neither scored nor failed. */
+    applies?: ScorerApplies<unknown, unknown, unknown>;
 };
 declare function llmScorer(config: LlmScorerConfig): Scorer;
 
@@ -209,6 +262,14 @@ type ScorerDetail = {
     metadata?: Record<string, unknown>;
     duration?: number;
     cost?: number;
+    /**
+     * `true` when the scorer's `applies` predicate returned `false` for this item,
+     * so the scorer was deliberately skipped (NOT run). Distinct from a `null`
+     * score WITH a `duration` (ran-and-failed) and from a `null` score with
+     * neither field (skipped by cancellation). A skipped item is excluded from the
+     * mean AND from the failure-rate denominator — see `scorerCounts`.
+     */
+    skipped?: boolean;
 };
 type EvalItem = {
     input: unknown;
@@ -254,6 +315,13 @@ type EvalSummary = {
          * means the `mean` was computed over a thinned sample.
          */
         failed?: number;
+        /**
+         * Number of items for which this scorer's `applies` predicate returned
+         * `false` — deliberately skipped, NOT run. Counted in neither `scored` nor
+         * `failed`, so it never inflates the failure rate. Optional so pre-existing
+         * artifacts stay valid; absent ⇒ recompute from `items` (0 if no skips).
+         */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -326,7 +394,16 @@ type EvalComparison = {
         significant?: boolean;
         pRegression?: number;
         pImprovement?: number;
-        /** Number of paired item differences used for CI computation. */
+        /**
+         * Paired sample size — the count of items scored on BOTH sides (the only
+         * items a difference exists for). Always set, including `0`/`1`. The CI /
+         * `significant` / `pRegression` / `pImprovement` fields are populated only
+         * when `n >= 2`. NOTE the asymmetry: `delta` is the difference of the two
+         * INDEPENDENT per-side means (each over `{baseline,candidate}Scored`
+         * items), whereas the CI is paired over these `n` — so when `n` is much
+         * smaller than the per-side scored counts (different skips/failures per
+         * side), the delta and the CI rest on different samples.
+         */
         n?: number;
         /**
          * Per-side scorer success/failure counts over the SAME truncated pool the
@@ -340,6 +417,16 @@ type EvalComparison = {
         baselineFailed?: number;
         candidateScored?: number;
         candidateFailed?: number;
+        /**
+         * Per-side `applies`-skipped (N/A) counts over the same truncated pool.
+         * Skips are excluded from both the mean and the failure-rate denominator,
+         * so they never move the delta — but a side that skipped items scored a
+         * different subset than a side that didn't, so surfacing the count lets a
+         * consumer explain a delta that's really a sample mismatch rather than a
+         * regression.
+         */
+        baselineSkipped?: number;
+        candidateSkipped?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -528,6 +615,9 @@ type MultiRunSummary = {
         scored?: number;
         /** Total ran-and-failed scorer invocations summed across all runs. */
         failed?: number;
+        /** Total `applies`-skipped (N/A) items summed across all runs. Excluded
+         *  from both the mean and the failure-rate denominator. */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -539,4 +629,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.d.ts

@@ -79,16 +79,64 @@ type ScorerContext = {
     signal?: AbortSignal;
 };
 type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
+/**
+ * Applicability predicate for a conditional scorer. Returns `true` if this
+ * scorer should run for the given item, `false` to skip it.
+ *
+ * The eval runner runs every scorer against every item, so a scorer that only
+ * applies to a subset (a refusal judge for refusal-expected items, a constraint
+ * judge for constrained items, …) declares its scope here. A skipped item is
+ * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND
+ * from the failure-rate denominator, so a conditional scorer stays honest to
+ * both. The predicate runs BEFORE the scorer body, so an inapplicable
+ * `llmScorer` never makes its provider call.
+ *
+ * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability
+ * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but
+ * `output` is available too (e.g. "only score outputs that refused"). A
+ * predicate that THROWS is a bug, not a skip — it's recorded as a scorer
+ * failure, never silently swallowed.
+ *
+ * The contract is `=> boolean`: return `true` to run, `false` to skip. The
+ * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy
+ * value (a predicate that forgot to `return`) skips rather than crashes — but
+ * authors should return a real boolean. When deriving from a truthy value,
+ * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not
+ * `=> a.constraints?.length` (which is `number | undefined`, a type error). A
+ * predicate that accidentally never returns therefore skips EVERY item — but
+ * that surfaces conspicuously as a non-zero `skipped` count (the Studio "N/A"
+ * chip), not as a silently green mean.
+ */
+type ScorerApplies<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => boolean;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
     score: ScorerFn<TOutput, TInput, TAnnotations>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the scorer is skipped for that item (counted as neither
+     *  scored nor failed). Omit to run on every item (the default).
+     *
+     *  Declared with **method syntax** (not a function-valued property) for parity
+     *  with {@link Scorer.applies}, where the syntax is load-bearing: see the note
+     *  there and `__tests__/scorer-assignability.test-d.ts`. */
+    applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;
 };
 type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
     score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
+    /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}.
+     *
+     *  Declared with **method syntax** (not a function-valued `ScorerApplies`
+     *  property) so its parameters are checked bivariantly, exactly like
+     *  {@link score}. `EvalConfig.scorers` erases the generics to
+     *  `Scorer<unknown, unknown, unknown>[]`; under `strictFunctionTypes` a
+     *  property would make those params contravariant, so a concretely-typed
+     *  `Scorer<…, TInput, TAnnotations>` would no longer be assignable to that
+     *  array (the 0.18.1 regression). `__tests__/scorer-assignability.test-d.ts`
+     *  locks this in and is compiled by the `typecheck` gate. */
+    applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -117,6 +165,11 @@ type LlmScorerConfig = {
     stop?: string[];
     /** Provider-specific escape hatch — merged last into the raw API request. */
     providerOptions?: Record<string, unknown>;
+    /** Optional applicability predicate — see {@link ScorerApplies}. When it
+     *  returns `false`, the judge is skipped for that item and NO provider call is
+     *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as
+     *  neither scored nor failed. */
+    applies?: ScorerApplies<unknown, unknown, unknown>;
 };
 declare function llmScorer(config: LlmScorerConfig): Scorer;
 
@@ -209,6 +262,14 @@ type ScorerDetail = {
     metadata?: Record<string, unknown>;
     duration?: number;
     cost?: number;
+    /**
+     * `true` when the scorer's `applies` predicate returned `false` for this item,
+     * so the scorer was deliberately skipped (NOT run). Distinct from a `null`
+     * score WITH a `duration` (ran-and-failed) and from a `null` score with
+     * neither field (skipped by cancellation). A skipped item is excluded from the
+     * mean AND from the failure-rate denominator — see `scorerCounts`.
+     */
+    skipped?: boolean;
 };
 type EvalItem = {
     input: unknown;
@@ -254,6 +315,13 @@ type EvalSummary = {
          * means the `mean` was computed over a thinned sample.
          */
         failed?: number;
+        /**
+         * Number of items for which this scorer's `applies` predicate returned
+         * `false` — deliberately skipped, NOT run. Counted in neither `scored` nor
+         * `failed`, so it never inflates the failure rate. Optional so pre-existing
+         * artifacts stay valid; absent ⇒ recompute from `items` (0 if no skips).
+         */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -326,7 +394,16 @@ type EvalComparison = {
         significant?: boolean;
         pRegression?: number;
         pImprovement?: number;
-        /** Number of paired item differences used for CI computation. */
+        /**
+         * Paired sample size — the count of items scored on BOTH sides (the only
+         * items a difference exists for). Always set, including `0`/`1`. The CI /
+         * `significant` / `pRegression` / `pImprovement` fields are populated only
+         * when `n >= 2`. NOTE the asymmetry: `delta` is the difference of the two
+         * INDEPENDENT per-side means (each over `{baseline,candidate}Scored`
+         * items), whereas the CI is paired over these `n` — so when `n` is much
+         * smaller than the per-side scored counts (different skips/failures per
+         * side), the delta and the CI rest on different samples.
+         */
         n?: number;
         /**
          * Per-side scorer success/failure counts over the SAME truncated pool the
@@ -340,6 +417,16 @@ type EvalComparison = {
         baselineFailed?: number;
         candidateScored?: number;
         candidateFailed?: number;
+        /**
+         * Per-side `applies`-skipped (N/A) counts over the same truncated pool.
+         * Skips are excluded from both the mean and the failure-rate denominator,
+         * so they never move the delta — but a side that skipped items scored a
+         * different subset than a side that didn't, so surfacing the count lets a
+         * consumer explain a delta that's really a sample mismatch rather than a
+         * regression.
+         */
+        baselineSkipped?: number;
+        candidateSkipped?: number;
     }>;
     timing?: {
         baselineMean: number;
@@ -528,6 +615,9 @@ type MultiRunSummary = {
         scored?: number;
         /** Total ran-and-failed scorer invocations summed across all runs. */
         failed?: number;
+        /** Total `applies`-skipped (N/A) items summed across all runs. Excluded
+         *  from both the mean and the failure-rate denominator. */
+        skipped?: number;
     }>;
     timing?: {
         mean: number;
@@ -539,4 +629,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   rescore,
   runEval,
   scorer
-} from "./chunk-BWBX5YVQ.js";
+} from "./chunk-4YWTVQ7T.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
@@ -113,6 +113,7 @@ function llmScorer(config) {
     name: config.name,
     description: config.description,
     isLlm: true,
+    applies: config.applies,
     async score(output, input, annotations, context) {
       if (!context?.resolveProvider) {
         throw new Error(

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AA2BtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACzIO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}
+{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AAgCtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;AC/IO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.18.0",
+  "version": "0.18.2",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.18.0"
+    "@axlsdk/axl": "0.18.2"
   },
   "peerDependencies": {
     "zod": "^4.0.0",