@axlsdk/eval 0.23.3 → 0.24.1

package/dist/index.js

@@ -1,15 +1,28 @@
 import {
+  DEFAULT_COPY_MAX_BYTES,
+  DEFAULT_SIDECAR_MAX_BYTES,
+  aggregateAccounting,
   aggregateRuns,
   attachScorerErrorCost,
   evalCompare,
+  evaluateItemErrorRateGate,
   evaluateScorerErrorRateGate,
   evaluateScorerTolerance,
+  isBudgetStopped,
   normalizeScorerResult,
   pairedBootstrapCI,
+  parseRequestRecords,
+  readAccounting,
+  refusedWork,
   rescore,
+  resolveCaptureLimits,
   runEval,
-  scorer
-} from "./chunk-AFUNZE7W.js";
+  scorer,
+  serializeRequestRecords,
+  toDiagnosticManifest,
+  unavailableManifest,
+  validateRequestSidecar
+} from "./chunk-HKEAPM4B.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
@@ -192,17 +205,30 @@ function defineEval(config) {
   return config;
 }
 export {
+  DEFAULT_COPY_MAX_BYTES,
+  DEFAULT_SIDECAR_MAX_BYTES,
+  aggregateAccounting,
   aggregateRuns,
   dataset,
   defineEval,
   evalCompare,
+  evaluateItemErrorRateGate,
   evaluateScorerErrorRateGate,
   evaluateScorerTolerance,
+  isBudgetStopped,
   llmScorer,
   normalizeScorerResult,
   pairedBootstrapCI,
+  parseRequestRecords,
+  readAccounting,
+  refusedWork,
   rescore,
+  resolveCaptureLimits,
   runEval,
-  scorer
+  scorer,
+  serializeRequestRecords,
+  toDiagnosticManifest,
+  unavailableManifest,
+  validateRequestSidecar
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\ntype ZodIssueLike = {\n  readonly path: readonly (string | number)[];\n  readonly message: string;\n};\n\nfunction isZodIssueLike(value: unknown): value is ZodIssueLike {\n  if (!value || typeof value !== 'object') return false;\n  const issue = value as { path?: unknown; message?: unknown };\n  return (\n    Array.isArray(issue.path) &&\n    issue.path.every((segment) => typeof segment === 'string' || typeof segment === 'number') &&\n    typeof issue.message === 'string'\n  );\n}\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        const issues = err && typeof err === 'object' && 'issues' in err ? err.issues : undefined;\n        if (Array.isArray(issues) && issues.every(isZodIssueLike)) {\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AAqC7C,SAAS,eAAe,OAAuC;AAC7D,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,QAAQ;AACd,SACE,MAAM,QAAQ,MAAM,IAAI,KACxB,MAAM,KAAK,MAAM,CAAC,YAAY,OAAO,YAAY,YAAY,OAAO,YAAY,QAAQ,KACxF,OAAO,MAAM,YAAY;AAE7B;AAEO,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,cAAM,SAAS,OAAO,OAAO,QAAQ,YAAY,YAAY,MAAM,IAAI,SAAS;AAChF,YAAI,MAAM,QAAQ,MAAM,KAAK,OAAO,MAAM,cAAc,GAAG;AACzD,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACtJO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}
+{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\ntype ZodIssueLike = {\n  readonly path: readonly (string | number)[];\n  readonly message: string;\n};\n\nfunction isZodIssueLike(value: unknown): value is ZodIssueLike {\n  if (!value || typeof value !== 'object') return false;\n  const issue = value as { path?: unknown; message?: unknown };\n  return (\n    Array.isArray(issue.path) &&\n    issue.path.every((segment) => typeof segment === 'string' || typeof segment === 'number') &&\n    typeof issue.message === 'string'\n  );\n}\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        const issues = err && typeof err === 'object' && 'issues' in err ? err.issues : undefined;\n        if (Array.isArray(issues) && issues.every(isZodIssueLike)) {\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AAqC7C,SAAS,eAAe,OAAuC;AAC7D,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,QAAQ;AACd,SACE,MAAM,QAAQ,MAAM,IAAI,KACxB,MAAM,KAAK,MAAM,CAAC,YAAY,OAAO,YAAY,YAAY,OAAO,YAAY,QAAQ,KACxF,OAAO,MAAM,YAAY;AAE7B;AAEO,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,cAAM,SAAS,OAAO,OAAO,QAAQ,YAAY,YAAY,MAAM,IAAI,SAAS;AAChF,YAAI,MAAM,QAAQ,MAAM,KAAK,OAAO,MAAM,cAAc,GAAG;AACzD,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACtJO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.23.3",
+  "version": "0.24.1",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.23.3"
+    "@axlsdk/axl": "0.24.1"
   },
   "peerDependencies": {
     "zod": "^4.0.0",