@axlsdk/eval 0.25.2 → 0.26.0

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\ntype ZodIssueLike = {\n  readonly path: readonly (string | number)[];\n  readonly message: string;\n};\n\nfunction isZodIssueLike(value: unknown): value is ZodIssueLike {\n  if (!value || typeof value !== 'object') return false;\n  const issue = value as { path?: unknown; message?: unknown };\n  return (\n    Array.isArray(issue.path) &&\n    issue.path.every((segment) => typeof segment === 'string' || typeof segment === 'number') &&\n    typeof issue.message === 'string'\n  );\n}\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        const issues = err && typeof err === 'object' && 'issues' in err ? err.issues : undefined;\n        if (Array.isArray(issues) && issues.every(isZodIssueLike)) {\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACxNA,SAAS,SAAS;AAGlB,SAAS,aAAa,uBAAuB;AAqC7C,SAAS,eAAe,OAAuC;AAC7D,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,QAAQ;AACd,SACE,MAAM,QAAQ,MAAM,IAAI,KACxB,MAAM,KAAK,MAAM,CAAC,YAAY,OAAO,YAAY,YAAY,OAAO,YAAY,QAAQ,KACxF,OAAO,MAAM,YAAY;AAE7B;AAEO,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,cAAM,SAAS,OAAO,OAAO,QAAQ,YAAY,YAAY,MAAM,IAAI,SAAS;AAChF,YAAI,MAAM,QAAQ,MAAM,KAAK,OAAO,MAAM,cAAc,GAAG;AACzD,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACtJO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path"]}
+{"version":3,"sources":["../src/dataset.ts","../src/model-calls.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\n/**\n * What to do when a dataset item's `annotations` contain keys that are absent\n * from the `annotations` schema.\n *\n * Zod object schemas strip unknown keys by default, so an annotation field\n * that isn't declared in the schema is silently dropped before it ever reaches\n * a scorer. A scorer that reads such a field sees `undefined` and quietly\n * degrades to a no-op — the eval still \"passes\", so the bug is invisible. This\n * is especially easy to hit with file-based datasets, where the JSON is never\n * type-checked.\n *\n * - `'warn'` (default) — log a single `console.warn` per dataset listing the\n *   dropped key paths. Non-breaking; surfaces the contract violation.\n * - `'error'` — throw, listing the dropped key paths. The strict path for CI.\n * - `'ignore'` — preserve the silent-strip behavior (skips detection entirely).\n *\n * Only `annotations` are checked, not `input`: stripped input fields flow to\n * the workflow where they surface, whereas stripped annotations vanish into a\n * silent scorer.\n */\nexport type ExtraKeyPolicy = 'warn' | 'error' | 'ignore';\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n  /**\n   * How to handle annotation keys not present in the `annotations` schema\n   * (Zod strips them by default). Defaults to `'warn'`. See {@link ExtraKeyPolicy}.\n   */\n  onExtraAnnotationKeys?: ExtraKeyPolicy;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  /**\n   * Annotation key paths the schema dropped during the most recent\n   * `getItems()` call (empty when none were dropped, or when\n   * `onExtraAnnotationKeys: 'ignore'`). Populated as a side effect of\n   * `getItems()` — read it synchronously right after awaiting, on the same\n   * instance; a later `getItems()` overwrites it (calling `getItems()`\n   * concurrently on one shared dataset is unsupported). The eval runner folds\n   * this into `EvalResult.metadata.droppedAnnotationKeys` so any consumer (the\n   * CLI logs it; UIs can render it) sees the same signal. Optional so\n   * hand-rolled `Dataset`-shaped objects (test harnesses) remain valid.\n   */\n  readonly droppedAnnotationKeys?: readonly string[];\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\n/** True for `{}`-style plain objects only — excludes arrays, null, Date, Map, class instances. */\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false;\n  const proto = Object.getPrototypeOf(value);\n  return proto === Object.prototype || proto === null;\n}\n\n/**\n * True when `schema` is a `ZodObject` (any unknown-key mode: strip/strict/loose).\n * Detected via the public `.shape` getter, which only `ZodObject` exposes —\n * transforms/pipes/preprocess (`.def.type === 'pipe'`), unions, records, arrays,\n * and wrappers (`.optional()` etc.) all return `undefined`. Uses duck-typing\n * rather than `instanceof z.ZodObject` so we keep a type-only zod import and\n * stay robust against duplicate zod copies in a consumer's tree.\n *\n * Drop-detection (raw-vs-parsed diff) is only sound when `.parse()` preserves\n * the input's object shape minus stripped keys. A `.transform()`/`.preprocess()`\n * that intentionally removes or reshapes keys would otherwise be mis-reported as\n * \"dropped by schema\" — so we gate detection on this predicate at the top level.\n */\nfunction isObjectSchema(schema: z.ZodType): boolean {\n  const shape = (schema as { shape?: unknown }).shape;\n  return typeof shape === 'object' && shape !== null;\n}\n\n/**\n * Recursively collect key paths present in `raw` but absent from `parsed`\n * (i.e. stripped by the schema). Walks plain objects and arrays; anything else\n * is a leaf. Paths use dot/bracket notation: `persona.tone`, `tags[0].label`.\n *\n * Only called for a top-level `ZodObject` annotations schema (see\n * `isObjectSchema`). Mode-independent within that scope — works whether the\n * object strips (detects drops), is strict (`.parse()` throws before we reach\n * here), or is loose (parsed keeps the keys, so nothing is reported). Known\n * limitation: a `.transform()` nested inside an object field that removes keys\n * can still produce a false positive for that sub-path; nested transforms on\n * annotation data are rare and not worth a full schema walk here.\n */\nfunction collectDroppedKeys(raw: unknown, parsed: unknown, prefix = ''): string[] {\n  if (Array.isArray(raw) && Array.isArray(parsed)) {\n    const dropped: string[] = [];\n    const n = Math.min(raw.length, parsed.length);\n    for (let i = 0; i < n; i++) {\n      dropped.push(...collectDroppedKeys(raw[i], parsed[i], `${prefix}[${i}]`));\n    }\n    return dropped;\n  }\n  if (isPlainObject(raw) && isPlainObject(parsed)) {\n    const dropped: string[] = [];\n    for (const key of Object.keys(raw)) {\n      const path = prefix ? `${prefix}.${key}` : key;\n      if (!(key in parsed)) {\n        dropped.push(path);\n      } else {\n        dropped.push(...collectDroppedKeys(raw[key], parsed[key], path));\n      }\n    }\n    return dropped;\n  }\n  return [];\n}\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  const policy: ExtraKeyPolicy = config.onExtraAnnotationKeys ?? 'warn';\n\n  /**\n   * Parse one item's input + annotations, recording any annotation keys the\n   * schema dropped into `droppedAcc` (deduped across the whole dataset so we\n   * emit a single warning rather than one per item).\n   */\n  function parseItem(\n    item: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>,\n    droppedAcc: Set<string>,\n  ): DatasetItem<z.infer<TInput>, z.infer<TAnnotations>> {\n    const input = config.schema.parse(item.input);\n    if (item.annotations == null || !config.annotations) {\n      return { input, annotations: item.annotations };\n    }\n    const annotations = config.annotations.parse(item.annotations);\n    // Only diff for plain object schemas: a transform/preprocess/pipe reshapes\n    // output, so a raw-vs-parsed diff would mis-report intentionally-removed\n    // keys as \"dropped by schema\" (and hard-fail under 'error').\n    if (policy !== 'ignore' && isObjectSchema(config.annotations)) {\n      for (const p of collectDroppedKeys(item.annotations, annotations)) droppedAcc.add(p);\n    }\n    return { input, annotations };\n  }\n\n  /** Emit the consolidated warning or throw, once per `getItems()` call. */\n  function reportDropped(dropped: Set<string>): void {\n    if (dropped.size === 0) return;\n    const keys = [...dropped].join(', ');\n    if (policy === 'error') {\n      throw new Error(\n        `Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n          `These keys are not declared in the annotations schema, so they were stripped and ` +\n          `will never reach scorers. Add them to the schema, or set ` +\n          `onExtraAnnotationKeys: 'ignore' to allow stripping.`,\n      );\n    }\n    console.warn(\n      `[axl-eval] Dataset \"${config.name}\": annotation key(s) dropped by schema: ${keys}. ` +\n        `These keys are not declared in the annotations schema, so they were stripped and ` +\n        `will not reach scorers (a common cause of no-op scorers). Add them to the schema, ` +\n        `set onExtraAnnotationKeys: 'error' to fail, or 'ignore' to silence this warning.`,\n    );\n  }\n\n  // Captured from the most recent getItems() so the runner can surface it\n  // structurally (in addition to the console.warn) without re-deriving it.\n  let lastDroppedAnnotationKeys: string[] = [];\n\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    get droppedAnnotationKeys() {\n      // Copy so the `readonly string[]` contract holds at runtime — a consumer\n      // can't mutate internal state, and a held reference can't observe a later\n      // getItems() reassignment.\n      return [...lastDroppedAnnotationKeys];\n    },\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n\n      let rawItems: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      if (config.items) {\n        rawItems = config.items;\n      } else {\n        const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n        const content = await readFile(resolvedPath, 'utf-8');\n        rawItems = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      }\n\n      const dropped = new Set<string>();\n      const parsed = rawItems.map((item) => parseItem(item, dropped));\n      // Set before reportDropped: in 'error' mode reportDropped throws, but the\n      // runner only reads the getter after a *successful* getItems(), so the\n      // value behind a rejected call is never observed.\n      lastDroppedAnnotationKeys = [...dropped];\n      reportDropped(dropped);\n      return parsed;\n    },\n  };\n}\n","/** Turn validated capture records into ordinary eval inputs and executors. */\nimport { readFile } from 'node:fs/promises';\nimport { z } from 'zod';\nimport {\n  AxlError,\n  capturedJournalLineSchema,\n  foldJournal,\n  foldedOperationRecordSchema,\n  modelCallRecordSchema,\n  type ChatMessage,\n  type ChatOptions,\n  type EvalExecuteWorkflow,\n  type FinishReason,\n  type FoldedOperationRecord,\n  type ModelCallRecord,\n  type ProviderResponse,\n  type StreamChunk,\n  type ToolCallMessage,\n} from '@axlsdk/axl';\nimport { dataset, type Dataset } from './dataset.js';\n\nexport type ModelCallInput = {\n  request?: ModelCallRecord['request'];\n  ids: ModelCallRecord['ids'];\n  attributes: ModelCallRecord['attributes'];\n  recorded: ModelCallRecord;\n};\n\nexport type ModelCallDatasetOptions<TAnnotations> = {\n  /** Runs after validation, including on external operations. */\n  filter?: (record: FoldedOperationRecord) => boolean;\n  /** Attach ground truth for scorers without changing their signature. */\n  annotate?: (record: ModelCallRecord, index: number) => TAnnotations;\n};\n\nexport type ReplayModelCallOptions = { model?: string };\n\n/**\n * The `output` both executors return: one model response, field-named like a\n * recorded `ok` outcome's `response` so a scorer can compare a replay with\n * `input.recorded` directly. Tool calls are part of the answer, not dropped.\n */\nexport type ModelCallOutput = {\n  content: string;\n  /** Parsed `content`, present when the request asked for JSON and the model answered in text. */\n  json?: unknown;\n  thinking?: string;\n  tool_calls?: ToolCallMessage[];\n  finishReason?: FinishReason;\n  rawFinishReason?: string;\n  responseId?: string;\n  responseModel?: string;\n};\n\ntype Located = { value: unknown; location: string; position: number };\nfunction invalid(location: string, message: string): never {\n  throw new AxlError('INVALID_MODEL_CALL_SOURCE', `${location}: ${message}`);\n}\nfunction versionOf(value: unknown): unknown {\n  return value !== null && typeof value === 'object'\n    ? Object.getOwnPropertyDescriptor(value, 'v')?.value\n    : undefined;\n}\nfunction parseJson(value: string, location: string): unknown {\n  try {\n    return JSON.parse(value) as unknown;\n  } catch {\n    return invalid(location, 'invalid JSON');\n  }\n}\nfunction arrayLength(source: unknown, location: string): number {\n  let array: boolean;\n  let descriptor: PropertyDescriptor | undefined;\n  try {\n    array = Array.isArray(source);\n    if (array) descriptor = Object.getOwnPropertyDescriptor(source, 'length');\n  } catch {\n    return invalid(location, 'array could not be inspected');\n  }\n  if (!array || !descriptor || !('value' in descriptor)) invalid(location, 'expected an array');\n  const length: unknown = descriptor.value;\n  if (typeof length !== 'number' || !Number.isSafeInteger(length) || length < 0)\n    invalid(location, 'invalid array length');\n  return length;\n}\nfunction arraySlot(source: readonly unknown[], index: number, location: string): unknown {\n  let descriptor: PropertyDescriptor | undefined;\n  try {\n    descriptor = Object.getOwnPropertyDescriptor(source, String(index));\n  } catch {\n    return invalid(location, 'array entry could not be inspected');\n  }\n  if (!descriptor || !('value' in descriptor))\n    invalid(location, 'array entry is missing or is an accessor');\n  return descriptor.value;\n}\nfunction* locateArray(\n  source: readonly unknown[],\n  at: (index: number) => string,\n): Iterable<Located> {\n  const length = arrayLength(source, 'source');\n  for (let index = 0; index < length; index++) {\n    const location = at(index);\n    yield { value: arraySlot(source, index, location), position: index, location };\n  }\n}\n\n/** Read v2 records or v1 phase journals without dropping external operations. */\nexport async function loadModelCalls(\n  source: string | readonly unknown[],\n): Promise<FoldedOperationRecord[]> {\n  let located: Iterable<Located>;\n  if (typeof source === 'string') {\n    const content = (await readFile(source, 'utf8')).replace(/^\\uFEFF/, '');\n    if (content.trimStart().startsWith('[')) {\n      const parsed = parseJson(content, `file ${source}`);\n      if (!Array.isArray(parsed)) invalid(`file ${source}`, 'expected a JSON array');\n      located = locateArray(parsed, (index) => `entry ${index + 1} of ${source}`);\n    } else {\n      const rows = content.split(/\\r?\\n/);\n      if (content.endsWith('\\n')) rows.pop(); // one final line terminator is not a blank record\n      located = rows.flatMap((row, index) => {\n        const location = `line ${index + 1} of ${source}`;\n        if (row.trim() === '') return [];\n        return [{ value: parseJson(row, location), position: index, location }];\n      });\n    }\n  } else {\n    located = locateArray(source, (index) => `item ${index + 1}`);\n  }\n\n  const journal: Array<Located & { operationId: string }> = [];\n  const folded: Array<{ record: FoldedOperationRecord; location: string; position: number }> = [];\n  const firstJournal = new Map<string, Located>();\n  let format: 1 | 2 | undefined;\n  for (const entry of located) {\n    let version: unknown;\n    try {\n      version = versionOf(entry.value);\n    } catch {\n      invalid(entry.location, 'record version could not be inspected');\n    }\n    if (version === 1 || version === 2) {\n      if (format !== undefined && format !== version)\n        invalid(entry.location, 'one source cannot mix v1 journal lines with v2 records');\n      format = version;\n    }\n    if (version === 1) {\n      const parsed = capturedJournalLineSchema.safeParse(entry.value);\n      if (!parsed.success) invalid(entry.location, parsed.error.message);\n      journal.push({ ...entry, value: parsed.data, operationId: parsed.data.operationId });\n      if (!firstJournal.has(parsed.data.operationId))\n        firstJournal.set(parsed.data.operationId, entry);\n    } else if (version === 2) {\n      const parsed = foldedOperationRecordSchema.safeParse(entry.value);\n      if (!parsed.success) invalid(entry.location, parsed.error.message);\n      folded.push({ record: parsed.data, location: entry.location, position: entry.position });\n    } else {\n      invalid(entry.location, 'unsupported capture version');\n    }\n  }\n  if (journal.length) {\n    let records: FoldedOperationRecord[];\n    try {\n      records = foldJournal(journal.map((entry) => entry.value));\n    } catch (error) {\n      const message = error instanceof Error ? error.message : String(error);\n      const line =\n        /\\bat line (\\d+)\\b/.exec(message) ?? /^Invalid capture journal line (\\d+):/.exec(message);\n      const indexed = line ? journal[Number(line[1]) - 1] : undefined;\n      const matched =\n        indexed ??\n        [...journal].reverse().find((entry) => {\n          const marker = `capture operation ${entry.operationId}`;\n          const at = message.indexOf(marker);\n          if (at < 0) return false;\n          const suffix = message.slice(at + marker.length);\n          return suffix === '' || suffix.startsWith(':') || suffix.startsWith(' at line');\n        });\n      invalid(matched?.location ?? journal[0].location, message);\n    }\n    for (const record of records) {\n      const first = firstJournal.get(record.id)!;\n      folded.push({ record, location: first.location, position: first.position });\n    }\n  }\n  folded.sort((a, b) => a.position - b.position);\n  const seen = new Map<string, string>();\n  for (const item of folded) {\n    const previous = seen.get(item.record.id);\n    if (previous)\n      invalid(item.location, `duplicate operation id ${item.record.id} (first at ${previous})`);\n    seen.set(item.record.id, item.location);\n  }\n  return folded.map((item) => item.record);\n}\n\nconst modelCallInputSchema = z.object({ recorded: modelCallRecordSchema }).transform(\n  ({ recorded }): ModelCallInput => ({\n    ...(recorded.request ? { request: recorded.request } : {}),\n    ids: recorded.ids,\n    attributes: recorded.attributes,\n    recorded,\n  }),\n);\n\n/** Create an ordinary Dataset for runEval/defineEval; external operations require an explicit filter. */\nexport function modelCallDataset<TAnnotations = undefined>(\n  records: readonly FoldedOperationRecord[],\n  options?: ModelCallDatasetOptions<TAnnotations>,\n): Dataset<ModelCallInput, TAnnotations> {\n  const items: Array<{ input: ModelCallInput; annotations?: TAnnotations }> = [];\n  const length = arrayLength(records, 'records');\n  for (let index = 0; index < length; index++) {\n    const parsed = foldedOperationRecordSchema.safeParse(\n      arraySlot(records, index, `record ${index + 1}`),\n    );\n    if (!parsed.success) invalid(`record ${index + 1}`, parsed.error.message);\n    const record = parsed.data;\n    if (options?.filter && !options.filter(record)) continue;\n    if (record.kind === 'external')\n      throw new AxlError(\n        'MODEL_CALL_EXPECTED',\n        `record ${index + 1} (${record.id}) is external; supply a filter to exclude it`,\n      );\n    const input: ModelCallInput = {\n      ...(record.request ? { request: record.request } : {}),\n      ids: record.ids,\n      attributes: record.attributes,\n      recorded: record,\n    };\n    items.push({\n      input,\n      ...(options?.annotate ? { annotations: options.annotate(record, index) } : {}),\n    });\n  }\n  return dataset({\n    name: 'model-calls',\n    schema: modelCallInputSchema,\n    annotations: z.custom<TAnnotations>(),\n    items,\n  });\n}\n\nfunction readInput(input: unknown): ModelCallRecord {\n  const candidate =\n    input !== null && typeof input === 'object'\n      ? Object.getOwnPropertyDescriptor(input, 'recorded')?.value\n      : undefined;\n  if (\n    candidate !== null &&\n    typeof candidate === 'object' &&\n    Object.getOwnPropertyDescriptor(candidate, 'kind')?.value === 'external'\n  )\n    throw new AxlError(\n      'MODEL_CALL_EXPECTED',\n      'external operations cannot be scored or replayed as model calls',\n    );\n  const parsed = modelCallRecordSchema.safeParse(candidate);\n  if (!parsed.success)\n    throw new AxlError(\n      'INVALID_MODEL_CALL_INPUT',\n      `expected a validated model-call input: ${parsed.error.message}`,\n    );\n  return parsed.data;\n}\ntype ResponseFields = Omit<ModelCallOutput, 'json'>;\nfunction modelCallOutput(response: ResponseFields, record: ModelCallRecord): ModelCallOutput {\n  const output: ModelCallOutput = {\n    content: response.content,\n    ...(response.thinking ? { thinking: response.thinking } : {}),\n    ...(response.tool_calls?.length ? { tool_calls: response.tool_calls } : {}),\n    ...(response.finishReason !== undefined ? { finishReason: response.finishReason } : {}),\n    ...(response.rawFinishReason !== undefined\n      ? { rawFinishReason: response.rawFinishReason }\n      : {}),\n    ...(response.responseId !== undefined ? { responseId: response.responseId } : {}),\n    ...(response.responseModel !== undefined ? { responseModel: response.responseModel } : {}),\n  };\n  const format = record.request?.responseFormat?.type;\n  if (format !== 'json_object' && format !== 'json_schema') return output;\n  // A tool-call turn under a JSON format legitimately carries no text yet.\n  if (output.content === '' && output.tool_calls) return output;\n  try {\n    return { ...output, json: JSON.parse(output.content) as unknown };\n  } catch {\n    throw new AxlError(\n      'INVALID_MODEL_CALL_JSON',\n      `model call ${record.id} did not return valid JSON`,\n    );\n  }\n}\nfunction providerOutput(response: ProviderResponse, record: ModelCallRecord): ModelCallOutput {\n  return modelCallOutput(\n    {\n      content: response.content,\n      ...(response.thinking_content !== undefined ? { thinking: response.thinking_content } : {}),\n      ...(response.tool_calls !== undefined ? { tool_calls: response.tool_calls } : {}),\n      ...(response.finishReason !== undefined ? { finishReason: response.finishReason } : {}),\n      ...(response.rawFinishReason !== undefined\n        ? { rawFinishReason: response.rawFinishReason }\n        : {}),\n      ...(response.responseId !== undefined ? { responseId: response.responseId } : {}),\n      ...(response.responseModel !== undefined ? { responseModel: response.responseModel } : {}),\n    },\n    record,\n  );\n}\n\n/** Rescore captured output (a {@link ModelCallOutput}) without calling a provider or returning historical cost. */\nexport function recordedModelCall(): EvalExecuteWorkflow {\n  return async (input) => {\n    const record = readInput(input);\n    if (record.captured.redacted)\n      throw new AxlError('RECORDED_OUTPUT_UNAVAILABLE', `model call ${record.id} was redacted`);\n    if (record.outcome.status !== 'ok' || !('response' in record.outcome))\n      throw new AxlError(\n        'RECORDED_OUTPUT_UNAVAILABLE',\n        `model call ${record.id} has no complete successful output (${record.outcome.status})`,\n      );\n    // Copies named response fields only; opaque providerMetadata is replay input, not output.\n    return { output: modelCallOutput(record.outcome.response, record) };\n  };\n}\n\nfunction replayRequest(record: ModelCallRecord): NonNullable<ModelCallRecord['request']> {\n  const request = record.request;\n  if (\n    !request ||\n    !record.captured.replayable ||\n    record.captured.continuityCaptured !== true ||\n    record.captured.redacted ||\n    record.captured.truncated ||\n    record.captured.omitted.length > 0 ||\n    (request.providerOptionKeys?.length ?? 0) > 0\n  )\n    throw new AxlError(\n      'MODEL_CALL_NOT_REPLAYABLE',\n      `model call ${record.id} does not contain an exact replayable request`,\n    );\n  return request;\n}\nfunction chatOptions(request: NonNullable<ModelCallRecord['request']>, model: string): ChatOptions {\n  if (!request.options.model)\n    throw new AxlError('INVALID_MODEL_CALL_INPUT', 'captured request model is empty');\n  const effort = request.options.effort;\n  if (effort !== undefined && !['none', 'low', 'medium', 'high', 'xhigh', 'max'].includes(effort))\n    throw new AxlError('INVALID_MODEL_CALL_INPUT', `unsupported captured effort ${effort}`);\n  const toolChoice = request.options.toolChoice;\n  if (\n    toolChoice !== undefined &&\n    toolChoice !== 'auto' &&\n    toolChoice !== 'none' &&\n    toolChoice !== 'required' &&\n    !(\n      toolChoice !== null &&\n      typeof toolChoice === 'object' &&\n      !Array.isArray(toolChoice) &&\n      toolChoice.type === 'function' &&\n      Object.keys(toolChoice).every((key) => key === 'type' || key === 'function') &&\n      toolChoice.function !== null &&\n      typeof toolChoice.function === 'object' &&\n      !Array.isArray(toolChoice.function) &&\n      typeof toolChoice.function.name === 'string' &&\n      Object.keys(toolChoice.function).every((key) => key === 'name')\n    )\n  )\n    throw new AxlError('INVALID_MODEL_CALL_INPUT', 'unsupported captured tool choice');\n  return {\n    model,\n    ...(request.options.temperature !== undefined\n      ? { temperature: request.options.temperature }\n      : {}),\n    ...(request.options.maxTokens !== undefined ? { maxTokens: request.options.maxTokens } : {}),\n    ...(request.options.thinkingBudget !== undefined\n      ? { thinkingBudget: request.options.thinkingBudget }\n      : {}),\n    ...(request.options.includeThoughts !== undefined\n      ? { includeThoughts: request.options.includeThoughts }\n      : {}),\n    ...(request.options.stop !== undefined ? { stop: [...request.options.stop] } : {}),\n    ...(request.options.promptCache !== undefined\n      ? { promptCache: request.options.promptCache }\n      : {}),\n    ...(effort !== undefined ? { effort: effort as ChatOptions['effort'] } : {}),\n    ...(toolChoice !== undefined ? { toolChoice: toolChoice as ChatOptions['toolChoice'] } : {}),\n    ...(request.tools\n      ? {\n          tools: request.tools.map((tool) => ({\n            type: 'function' as const,\n            function: {\n              name: tool.name,\n              description: tool.description ?? '',\n              parameters: tool.parameters,\n              ...(tool.strict !== undefined ? { strict: tool.strict } : {}),\n            },\n          })),\n        }\n      : {}),\n    ...(request.responseFormat ? { responseFormat: request.responseFormat } : {}),\n  };\n}\nfunction chatMessages(request: NonNullable<ModelCallRecord['request']>): ChatMessage[] {\n  return request.messages.map((message) => {\n    if (!['system', 'user', 'assistant', 'tool'].includes(message.role))\n      throw new AxlError('MODEL_CALL_NOT_REPLAYABLE', 'captured message cannot be reconstructed');\n    let content: ChatMessage['content'];\n    if (message.input) {\n      const text = message.content ?? '';\n      let offset = 0;\n      const parts: Array<{ type: 'text'; text: string }> = [];\n      for (const part of message.input.parts) {\n        if (part.type !== 'text' || offset + part.characters > text.length)\n          throw new AxlError(\n            'MODEL_CALL_NOT_REPLAYABLE',\n            'captured content parts cannot be reconstructed',\n          );\n        parts.push({ type: 'text', text: text.slice(offset, offset + part.characters) });\n        offset += part.characters;\n      }\n      if (offset !== text.length)\n        throw new AxlError(\n          'MODEL_CALL_NOT_REPLAYABLE',\n          'captured text lengths do not match content',\n        );\n      content = parts;\n    } else if (message.content !== null) {\n      content = message.content;\n    } else {\n      throw new AxlError('MODEL_CALL_NOT_REPLAYABLE', 'captured message content is absent');\n    }\n    return {\n      role: message.role as ChatMessage['role'],\n      content,\n      ...(message.name !== undefined ? { name: message.name } : {}),\n      ...(message.agent !== undefined ? { agent: message.agent } : {}),\n      ...(message.origin !== undefined ? { origin: message.origin } : {}),\n      ...(message.tool_calls !== undefined ? { tool_calls: message.tool_calls } : {}),\n      ...(message.tool_call_id !== undefined ? { tool_call_id: message.tool_call_id } : {}),\n      ...(message.providerMetadata !== undefined\n        ? { providerMetadata: message.providerMetadata }\n        : {}),\n    };\n  });\n}\n\n/** Replay one captured provider call into a {@link ModelCallOutput}; returned tool calls are not executed. */\nexport function replayModelCall(options: ReplayModelCallOptions = {}): EvalExecuteWorkflow {\n  return async (input, runtime) => {\n    const record = readInput(input);\n    const request = replayRequest(record); // reject before resolving or dispatching\n    const messages = chatMessages(request);\n    const capturedOptions = chatOptions(request, request.options.model);\n    if (!runtime)\n      throw new AxlError('MODEL_CALL_RUNTIME_REQUIRED', 'replay requires an AxlRuntime');\n    const uri = options.model ?? `${record.provider}:${request.options.model}`;\n    if (!uri.includes(':'))\n      throw new AxlError('MODEL_CALL_MODEL_UNRESOLVED', 'model override must use provider:model');\n    let resolved: ReturnType<typeof runtime.resolveProvider>;\n    try {\n      resolved = runtime.resolveProvider(uri);\n    } catch {\n      throw new AxlError(\n        'MODEL_CALL_MODEL_UNRESOLVED',\n        `cannot resolve ${uri}; supply an explicit provider:model override`,\n      );\n    }\n    if (!options.model && resolved.provider.name !== record.provider)\n      throw new AxlError(\n        'MODEL_CALL_MODEL_UNRESOLVED',\n        `resolved ${uri} to a different provider; supply an explicit override`,\n      );\n    const callOptions = {\n      ...capturedOptions,\n      model: options.model ? resolved.model : request.options.model,\n    };\n    if (record.kind === 'chat') {\n      const response = await resolved.provider.chat(messages, callOptions);\n      return { output: providerOutput(response, record) };\n    }\n    let content = '';\n    let thinking = '';\n    // Same accumulation as the ask loop: deltas keyed by id, name set, arguments appended.\n    const calls = new Map<string, { id: string; name: string; arguments: string }>();\n    let done: Extract<StreamChunk, { type: 'done' }> | undefined;\n    for await (const chunk of resolved.provider.stream(messages, callOptions)) {\n      if (chunk.type === 'text_delta') content += chunk.content;\n      else if (chunk.type === 'thinking_delta') thinking += chunk.content;\n      else if (chunk.type === 'tool_call_delta') {\n        const call = calls.get(chunk.id) ?? { id: chunk.id, name: '', arguments: '' };\n        if (chunk.name) call.name = chunk.name;\n        if (chunk.arguments) call.arguments += chunk.arguments;\n        calls.set(chunk.id, call);\n      } else if (chunk.type === 'done') {\n        done = chunk;\n        break;\n      }\n    }\n    if (!done)\n      throw new AxlError('MODEL_CALL_STREAM_INCOMPLETE', 'replayed stream ended without done');\n    return {\n      output: providerOutput(\n        {\n          content,\n          ...(thinking ? { thinking_content: thinking } : {}),\n          ...(calls.size\n            ? {\n                tool_calls: [...calls.values()].map((call) => ({\n                  id: call.id,\n                  type: 'function' as const,\n                  function: { name: call.name, arguments: call.arguments },\n                })),\n              }\n            : {}),\n          ...(done.finishReason !== undefined ? { finishReason: done.finishReason } : {}),\n          ...(done.rawFinishReason !== undefined ? { rawFinishReason: done.rawFinishReason } : {}),\n          ...(done.responseId !== undefined ? { responseId: done.responseId } : {}),\n          ...(done.responseModel !== undefined ? { responseModel: done.responseModel } : {}),\n        },\n        record,\n      ),\n    };\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerApplies, ScorerContext, ScorerResult } from './scorer.js';\nimport { attachScorerErrorCost } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\nimport type { Effort } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  /** Sampling temperature. Default: 0.2 for deterministic judging. */\n  temperature?: number;\n  /** Cap on response tokens. Falls back to provider default if unset. */\n  maxTokens?: number;\n  /** Reasoning effort — `'high'` or `'max'` materially improves judge calibration on\n   *  reasoning-capable models (gpt-5.x, Opus 4.6+, Gemini 3.x). See `Effort` for\n   *  provider-specific mapping. */\n  effort?: Effort;\n  /** Precise thinking-token budget override. See `ChatOptions.thinkingBudget`. */\n  thinkingBudget?: number;\n  /** Surface reasoning summaries in the judge's response (Gemini + OpenAI Responses). */\n  includeThoughts?: boolean;\n  /** Stop sequences forwarded to the provider. */\n  stop?: string[];\n  /** Provider-specific escape hatch — merged last into the raw API request. */\n  providerOptions?: Record<string, unknown>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the judge is skipped for that item and NO provider call is\n   *  made (the conditional-LLM-judge cost/rate-limit win). The item counts as\n   *  neither scored nor failed. */\n  applies?: ScorerApplies<unknown, unknown, unknown>;\n};\n\ntype ZodIssueLike = {\n  readonly path: readonly (string | number)[];\n  readonly message: string;\n};\n\nfunction isZodIssueLike(value: unknown): value is ZodIssueLike {\n  if (!value || typeof value !== 'object') return false;\n  const issue = value as { path?: unknown; message?: unknown };\n  return (\n    Array.isArray(issue.path) &&\n    issue.path.every((segment) => typeof segment === 'string' || typeof segment === 'number') &&\n    typeof issue.message === 'string'\n  );\n}\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    applies: config.applies,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      // Field-by-field forwarding (deliberately NOT spreading `config`):\n      // `LlmScorerConfig` has scorer-specific fields (`name`, `description`,\n      // `schema`, `system`, `model` — the URI, not the chat-options `model`\n      // which is the post-resolveProvider stripped form) that must not leak\n      // into ChatOptions. A spread would silently pass them through to\n      // provider.chat and either be ignored or, worse, override our hardcoded\n      // `responseFormat`. Keep this explicit.\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        {\n          model,\n          temperature: config.temperature ?? 0.2,\n          maxTokens: config.maxTokens,\n          effort: config.effort,\n          thinkingBudget: config.thinkingBudget,\n          includeThoughts: config.includeThoughts,\n          stop: config.stop,\n          providerOptions: config.providerOptions,\n          responseFormat: { type: 'json_object' },\n          signal: context.signal,\n        },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        attachScorerErrorCost(err, responseCost);\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        const issues = err && typeof err === 'object' && 'issues' in err ? err.issues : undefined;\n        if (Array.isArray(issues) && issues.every(isZodIssueLike)) {\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          attachScorerErrorCost(error, responseCost);\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAgEtB,SAAS,cAAc,OAAkD;AACvE,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AAChF,QAAM,QAAQ,OAAO,eAAe,KAAK;AACzC,SAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAeA,SAAS,eAAe,QAA4B;AAClD,QAAM,QAAS,OAA+B;AAC9C,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAeA,SAAS,mBAAmB,KAAc,QAAiB,SAAS,IAAc;AAChF,MAAI,MAAM,QAAQ,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,UAAM,IAAI,KAAK,IAAI,IAAI,QAAQ,OAAO,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,cAAQ,KAAK,GAAG,mBAAmB,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACA,MAAI,cAAc,GAAG,KAAK,cAAc,MAAM,GAAG;AAC/C,UAAM,UAAoB,CAAC;AAC3B,eAAW,OAAO,OAAO,KAAK,GAAG,GAAG;AAClC,YAAMA,QAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,UAAI,EAAE,OAAO,SAAS;AACpB,gBAAQ,KAAKA,KAAI;AAAA,MACnB,OAAO;AACL,gBAAQ,KAAK,GAAG,mBAAmB,IAAI,GAAG,GAAG,OAAO,GAAG,GAAGA,KAAI,CAAC;AAAA,MACjE;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEO,SAAS,QACd,QACiD;AACjD,QAAM,SAAyB,OAAO,yBAAyB;AAO/D,WAAS,UACP,MACA,YACqD;AACrD,UAAM,QAAQ,OAAO,OAAO,MAAM,KAAK,KAAK;AAC5C,QAAI,KAAK,eAAe,QAAQ,CAAC,OAAO,aAAa;AACnD,aAAO,EAAE,OAAO,aAAa,KAAK,YAAY;AAAA,IAChD;AACA,UAAM,cAAc,OAAO,YAAY,MAAM,KAAK,WAAW;AAI7D,QAAI,WAAW,YAAY,eAAe,OAAO,WAAW,GAAG;AAC7D,iBAAW,KAAK,mBAAmB,KAAK,aAAa,WAAW,EAAG,YAAW,IAAI,CAAC;AAAA,IACrF;AACA,WAAO,EAAE,OAAO,YAAY;AAAA,EAC9B;AAGA,WAAS,cAAc,SAA4B;AACjD,QAAI,QAAQ,SAAS,EAAG;AACxB,UAAM,OAAO,CAAC,GAAG,OAAO,EAAE,KAAK,IAAI;AACnC,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO,IAAI,2CAA2C,IAAI;AAAA,MAIxE;AAAA,IACF;AACA,YAAQ;AAAA,MACN,uBAAuB,OAAO,IAAI,2CAA2C,IAAI;AAAA,IAInF;AAAA,EACF;AAIA,MAAI,4BAAsC,CAAC;AAE3C,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,IAAI,wBAAwB;AAI1B,aAAO,CAAC,GAAG,yBAAyB;AAAA,IACtC;AAAA,IACA,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AAEA,UAAI;AACJ,UAAI,OAAO,OAAO;AAChB,mBAAW,OAAO;AAAA,MACpB,OAAO;AACL,cAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,cAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,mBAAW,KAAK,MAAM,OAAO;AAAA,MAC/B;AAEA,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,SAAS,SAAS,IAAI,CAAC,SAAS,UAAU,MAAM,OAAO,CAAC;AAI9D,kCAA4B,CAAC,GAAG,OAAO;AACvC,oBAAc,OAAO;AACrB,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACvNA,SAAS,YAAAC,iBAAgB;AACzB,SAAS,SAAS;AAClB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAUK;AAqCP,SAAS,QAAQ,UAAkB,SAAwB;AACzD,QAAM,IAAI,SAAS,6BAA6B,GAAG,QAAQ,KAAK,OAAO,EAAE;AAC3E;AACA,SAAS,UAAU,OAAyB;AAC1C,SAAO,UAAU,QAAQ,OAAO,UAAU,WACtC,OAAO,yBAAyB,OAAO,GAAG,GAAG,QAC7C;AACN;AACA,SAAS,UAAU,OAAe,UAA2B;AAC3D,MAAI;AACF,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB,QAAQ;AACN,WAAO,QAAQ,UAAU,cAAc;AAAA,EACzC;AACF;AACA,SAAS,YAAY,QAAiB,UAA0B;AAC9D,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,YAAQ,MAAM,QAAQ,MAAM;AAC5B,QAAI,MAAO,cAAa,OAAO,yBAAyB,QAAQ,QAAQ;AAAA,EAC1E,QAAQ;AACN,WAAO,QAAQ,UAAU,8BAA8B;AAAA,EACzD;AACA,MAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,YAAa,SAAQ,UAAU,mBAAmB;AAC5F,QAAM,SAAkB,WAAW;AACnC,MAAI,OAAO,WAAW,YAAY,CAAC,OAAO,cAAc,MAAM,KAAK,SAAS;AAC1E,YAAQ,UAAU,sBAAsB;AAC1C,SAAO;AACT;AACA,SAAS,UAAU,QAA4B,OAAe,UAA2B;AACvF,MAAI;AACJ,MAAI;AACF,iBAAa,OAAO,yBAAyB,QAAQ,OAAO,KAAK,CAAC;AAAA,EACpE,QAAQ;AACN,WAAO,QAAQ,UAAU,oCAAoC;AAAA,EAC/D;AACA,MAAI,CAAC,cAAc,EAAE,WAAW;AAC9B,YAAQ,UAAU,0CAA0C;AAC9D,SAAO,WAAW;AACpB;AACA,UAAU,YACR,QACA,IACmB;AACnB,QAAM,SAAS,YAAY,QAAQ,QAAQ;AAC3C,WAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS;AAC3C,UAAM,WAAW,GAAG,KAAK;AACzB,UAAM,EAAE,OAAO,UAAU,QAAQ,OAAO,QAAQ,GAAG,UAAU,OAAO,SAAS;AAAA,EAC/E;AACF;AAGA,eAAsB,eACpB,QACkC;AAClC,MAAI;AACJ,MAAI,OAAO,WAAW,UAAU;AAC9B,UAAM,WAAW,MAAMC,UAAS,QAAQ,MAAM,GAAG,QAAQ,WAAW,EAAE;AACtE,QAAI,QAAQ,UAAU,EAAE,WAAW,GAAG,GAAG;AACvC,YAAM,SAAS,UAAU,SAAS,QAAQ,MAAM,EAAE;AAClD,UAAI,CAAC,MAAM,QAAQ,MAAM,EAAG,SAAQ,QAAQ,MAAM,IAAI,uBAAuB;AAC7E,gBAAU,YAAY,QAAQ,CAAC,UAAU,SAAS,QAAQ,CAAC,OAAO,MAAM,EAAE;AAAA,IAC5E,OAAO;AACL,YAAM,OAAO,QAAQ,MAAM,OAAO;AAClC,UAAI,QAAQ,SAAS,IAAI,EAAG,MAAK,IAAI;AACrC,gBAAU,KAAK,QAAQ,CAAC,KAAK,UAAU;AACrC,cAAM,WAAW,QAAQ,QAAQ,CAAC,OAAO,MAAM;AAC/C,YAAI,IAAI,KAAK,MAAM,GAAI,QAAO,CAAC;AAC/B,eAAO,CAAC,EAAE,OAAO,UAAU,KAAK,QAAQ,GAAG,UAAU,OAAO,SAAS,CAAC;AAAA,MACxE,CAAC;AAAA,IACH;AAAA,EACF,OAAO;AACL,cAAU,YAAY,QAAQ,CAAC,UAAU,QAAQ,QAAQ,CAAC,EAAE;AAAA,EAC9D;AAEA,QAAM,UAAoD,CAAC;AAC3D,QAAM,SAAuF,CAAC;AAC9F,QAAM,eAAe,oBAAI,IAAqB;AAC9C,MAAI;AACJ,aAAW,SAAS,SAAS;AAC3B,QAAI;AACJ,QAAI;AACF,gBAAU,UAAU,MAAM,KAAK;AAAA,IACjC,QAAQ;AACN,cAAQ,MAAM,UAAU,uCAAuC;AAAA,IACjE;AACA,QAAI,YAAY,KAAK,YAAY,GAAG;AAClC,UAAI,WAAW,UAAa,WAAW;AACrC,gBAAQ,MAAM,UAAU,wDAAwD;AAClF,eAAS;AAAA,IACX;AACA,QAAI,YAAY,GAAG;AACjB,YAAM,SAAS,0BAA0B,UAAU,MAAM,KAAK;AAC9D,UAAI,CAAC,OAAO,QAAS,SAAQ,MAAM,UAAU,OAAO,MAAM,OAAO;AACjE,cAAQ,KAAK,EAAE,GAAG,OAAO,OAAO,OAAO,MAAM,aAAa,OAAO,KAAK,YAAY,CAAC;AACnF,UAAI,CAAC,aAAa,IAAI,OAAO,KAAK,WAAW;AAC3C,qBAAa,IAAI,OAAO,KAAK,aAAa,KAAK;AAAA,IACnD,WAAW,YAAY,GAAG;AACxB,YAAM,SAAS,4BAA4B,UAAU,MAAM,KAAK;AAChE,UAAI,CAAC,OAAO,QAAS,SAAQ,MAAM,UAAU,OAAO,MAAM,OAAO;AACjE,aAAO,KAAK,EAAE,QAAQ,OAAO,MAAM,UAAU,MAAM,UAAU,UAAU,MAAM,SAAS,CAAC;AAAA,IACzF,OAAO;AACL,cAAQ,MAAM,UAAU,6BAA6B;AAAA,IACvD;AAAA,EACF;AACA,MAAI,QAAQ,QAAQ;AAClB,QAAI;AACJ,QAAI;AACF,gBAAU,YAAY,QAAQ,IAAI,CAAC,UAAU,MAAM,KAAK,CAAC;AAAA,IAC3D,SAAS,OAAO;AACd,YAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,YAAM,OACJ,oBAAoB,KAAK,OAAO,KAAK,uCAAuC,KAAK,OAAO;AAC1F,YAAM,UAAU,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI;AACtD,YAAM,UACJ,WACA,CAAC,GAAG,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU;AACrC,cAAM,SAAS,qBAAqB,MAAM,WAAW;AACrD,cAAM,KAAK,QAAQ,QAAQ,MAAM;AACjC,YAAI,KAAK,EAAG,QAAO;AACnB,cAAM,SAAS,QAAQ,MAAM,KAAK,OAAO,MAAM;AAC/C,eAAO,WAAW,MAAM,OAAO,WAAW,GAAG,KAAK,OAAO,WAAW,UAAU;AAAA,MAChF,CAAC;AACH,cAAQ,SAAS,YAAY,QAAQ,CAAC,EAAE,UAAU,OAAO;AAAA,IAC3D;AACA,eAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,aAAa,IAAI,OAAO,EAAE;AACxC,aAAO,KAAK,EAAE,QAAQ,UAAU,MAAM,UAAU,UAAU,MAAM,SAAS,CAAC;AAAA,IAC5E;AAAA,EACF;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAC7C,QAAM,OAAO,oBAAI,IAAoB;AACrC,aAAW,QAAQ,QAAQ;AACzB,UAAM,WAAW,KAAK,IAAI,KAAK,OAAO,EAAE;AACxC,QAAI;AACF,cAAQ,KAAK,UAAU,0BAA0B,KAAK,OAAO,EAAE,cAAc,QAAQ,GAAG;AAC1F,SAAK,IAAI,KAAK,OAAO,IAAI,KAAK,QAAQ;AAAA,EACxC;AACA,SAAO,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM;AACzC;AAEA,IAAM,uBAAuB,EAAE,OAAO,EAAE,UAAU,sBAAsB,CAAC,EAAE;AAAA,EACzE,CAAC,EAAE,SAAS,OAAuB;AAAA,IACjC,GAAI,SAAS,UAAU,EAAE,SAAS,SAAS,QAAQ,IAAI,CAAC;AAAA,IACxD,KAAK,SAAS;AAAA,IACd,YAAY,SAAS;AAAA,IACrB;AAAA,EACF;AACF;AAGO,SAAS,iBACd,SACA,SACuC;AACvC,QAAM,QAAsE,CAAC;AAC7E,QAAM,SAAS,YAAY,SAAS,SAAS;AAC7C,WAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS;AAC3C,UAAM,SAAS,4BAA4B;AAAA,MACzC,UAAU,SAAS,OAAO,UAAU,QAAQ,CAAC,EAAE;AAAA,IACjD;AACA,QAAI,CAAC,OAAO,QAAS,SAAQ,UAAU,QAAQ,CAAC,IAAI,OAAO,MAAM,OAAO;AACxE,UAAM,SAAS,OAAO;AACtB,QAAI,SAAS,UAAU,CAAC,QAAQ,OAAO,MAAM,EAAG;AAChD,QAAI,OAAO,SAAS;AAClB,YAAM,IAAI;AAAA,QACR;AAAA,QACA,UAAU,QAAQ,CAAC,KAAK,OAAO,EAAE;AAAA,MACnC;AACF,UAAM,QAAwB;AAAA,MAC5B,GAAI,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,MACpD,KAAK,OAAO;AAAA,MACZ,YAAY,OAAO;AAAA,MACnB,UAAU;AAAA,IACZ;AACA,UAAM,KAAK;AAAA,MACT;AAAA,MACA,GAAI,SAAS,WAAW,EAAE,aAAa,QAAQ,SAAS,QAAQ,KAAK,EAAE,IAAI,CAAC;AAAA,IAC9E,CAAC;AAAA,EACH;AACA,SAAO,QAAQ;AAAA,IACb,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,aAAa,EAAE,OAAqB;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAEA,SAAS,UAAU,OAAiC;AAClD,QAAM,YACJ,UAAU,QAAQ,OAAO,UAAU,WAC/B,OAAO,yBAAyB,OAAO,UAAU,GAAG,QACpD;AACN,MACE,cAAc,QACd,OAAO,cAAc,YACrB,OAAO,yBAAyB,WAAW,MAAM,GAAG,UAAU;AAE9D,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AACF,QAAM,SAAS,sBAAsB,UAAU,SAAS;AACxD,MAAI,CAAC,OAAO;AACV,UAAM,IAAI;AAAA,MACR;AAAA,MACA,0CAA0C,OAAO,MAAM,OAAO;AAAA,IAChE;AACF,SAAO,OAAO;AAChB;AAEA,SAAS,gBAAgB,UAA0B,QAA0C;AAC3F,QAAM,SAA0B;AAAA,IAC9B,SAAS,SAAS;AAAA,IAClB,GAAI,SAAS,WAAW,EAAE,UAAU,SAAS,SAAS,IAAI,CAAC;AAAA,IAC3D,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;AAAA,IACzE,GAAI,SAAS,iBAAiB,SAAY,EAAE,cAAc,SAAS,aAAa,IAAI,CAAC;AAAA,IACrF,GAAI,SAAS,oBAAoB,SAC7B,EAAE,iBAAiB,SAAS,gBAAgB,IAC5C,CAAC;AAAA,IACL,GAAI,SAAS,eAAe,SAAY,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;AAAA,IAC/E,GAAI,SAAS,kBAAkB,SAAY,EAAE,eAAe,SAAS,cAAc,IAAI,CAAC;AAAA,EAC1F;AACA,QAAM,SAAS,OAAO,SAAS,gBAAgB;AAC/C,MAAI,WAAW,iBAAiB,WAAW,cAAe,QAAO;AAEjE,MAAI,OAAO,YAAY,MAAM,OAAO,WAAY,QAAO;AACvD,MAAI;AACF,WAAO,EAAE,GAAG,QAAQ,MAAM,KAAK,MAAM,OAAO,OAAO,EAAa;AAAA,EAClE,QAAQ;AACN,UAAM,IAAI;AAAA,MACR;AAAA,MACA,cAAc,OAAO,EAAE;AAAA,IACzB;AAAA,EACF;AACF;AACA,SAAS,eAAe,UAA4B,QAA0C;AAC5F,SAAO;AAAA,IACL;AAAA,MACE,SAAS,SAAS;AAAA,MAClB,GAAI,SAAS,qBAAqB,SAAY,EAAE,UAAU,SAAS,iBAAiB,IAAI,CAAC;AAAA,MACzF,GAAI,SAAS,eAAe,SAAY,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;AAAA,MAC/E,GAAI,SAAS,iBAAiB,SAAY,EAAE,cAAc,SAAS,aAAa,IAAI,CAAC;AAAA,MACrF,GAAI,SAAS,oBAAoB,SAC7B,EAAE,iBAAiB,SAAS,gBAAgB,IAC5C,CAAC;AAAA,MACL,GAAI,SAAS,eAAe,SAAY,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;AAAA,MAC/E,GAAI,SAAS,kBAAkB,SAAY,EAAE,eAAe,SAAS,cAAc,IAAI,CAAC;AAAA,IAC1F;AAAA,IACA;AAAA,EACF;AACF;AAGO,SAAS,oBAAyC;AACvD,SAAO,OAAO,UAAU;AACtB,UAAM,SAAS,UAAU,KAAK;AAC9B,QAAI,OAAO,SAAS;AAClB,YAAM,IAAI,SAAS,+BAA+B,cAAc,OAAO,EAAE,eAAe;AAC1F,QAAI,OAAO,QAAQ,WAAW,QAAQ,EAAE,cAAc,OAAO;AAC3D,YAAM,IAAI;AAAA,QACR;AAAA,QACA,cAAc,OAAO,EAAE,uCAAuC,OAAO,QAAQ,MAAM;AAAA,MACrF;AAEF,WAAO,EAAE,QAAQ,gBAAgB,OAAO,QAAQ,UAAU,MAAM,EAAE;AAAA,EACpE;AACF;AAEA,SAAS,cAAc,QAAkE;AACvF,QAAM,UAAU,OAAO;AACvB,MACE,CAAC,WACD,CAAC,OAAO,SAAS,cACjB,OAAO,SAAS,uBAAuB,QACvC,OAAO,SAAS,YAChB,OAAO,SAAS,aAChB,OAAO,SAAS,QAAQ,SAAS,MAChC,QAAQ,oBAAoB,UAAU,KAAK;AAE5C,UAAM,IAAI;AAAA,MACR;AAAA,MACA,cAAc,OAAO,EAAE;AAAA,IACzB;AACF,SAAO;AACT;AACA,SAAS,YAAY,SAAkD,OAA4B;AACjG,MAAI,CAAC,QAAQ,QAAQ;AACnB,UAAM,IAAI,SAAS,4BAA4B,iCAAiC;AAClF,QAAM,SAAS,QAAQ,QAAQ;AAC/B,MAAI,WAAW,UAAa,CAAC,CAAC,QAAQ,OAAO,UAAU,QAAQ,SAAS,KAAK,EAAE,SAAS,MAAM;AAC5F,UAAM,IAAI,SAAS,4BAA4B,+BAA+B,MAAM,EAAE;AACxF,QAAM,aAAa,QAAQ,QAAQ;AACnC,MACE,eAAe,UACf,eAAe,UACf,eAAe,UACf,eAAe,cACf,EACE,eAAe,QACf,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,KACzB,WAAW,SAAS,cACpB,OAAO,KAAK,UAAU,EAAE,MAAM,CAAC,QAAQ,QAAQ,UAAU,QAAQ,UAAU,KAC3E,WAAW,aAAa,QACxB,OAAO,WAAW,aAAa,YAC/B,CAAC,MAAM,QAAQ,WAAW,QAAQ,KAClC,OAAO,WAAW,SAAS,SAAS,YACpC,OAAO,KAAK,WAAW,QAAQ,EAAE,MAAM,CAAC,QAAQ,QAAQ,MAAM;AAGhE,UAAM,IAAI,SAAS,4BAA4B,kCAAkC;AACnF,SAAO;AAAA,IACL;AAAA,IACA,GAAI,QAAQ,QAAQ,gBAAgB,SAChC,EAAE,aAAa,QAAQ,QAAQ,YAAY,IAC3C,CAAC;AAAA,IACL,GAAI,QAAQ,QAAQ,cAAc,SAAY,EAAE,WAAW,QAAQ,QAAQ,UAAU,IAAI,CAAC;AAAA,IAC1F,GAAI,QAAQ,QAAQ,mBAAmB,SACnC,EAAE,gBAAgB,QAAQ,QAAQ,eAAe,IACjD,CAAC;AAAA,IACL,GAAI,QAAQ,QAAQ,oBAAoB,SACpC,EAAE,iBAAiB,QAAQ,QAAQ,gBAAgB,IACnD,CAAC;AAAA,IACL,GAAI,QAAQ,QAAQ,SAAS,SAAY,EAAE,MAAM,CAAC,GAAG,QAAQ,QAAQ,IAAI,EAAE,IAAI,CAAC;AAAA,IAChF,GAAI,QAAQ,QAAQ,gBAAgB,SAChC,EAAE,aAAa,QAAQ,QAAQ,YAAY,IAC3C,CAAC;AAAA,IACL,GAAI,WAAW,SAAY,EAAE,OAAwC,IAAI,CAAC;AAAA,IAC1E,GAAI,eAAe,SAAY,EAAE,WAAoD,IAAI,CAAC;AAAA,IAC1F,GAAI,QAAQ,QACR;AAAA,MACE,OAAO,QAAQ,MAAM,IAAI,CAAC,UAAU;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,UACR,MAAM,KAAK;AAAA,UACX,aAAa,KAAK,eAAe;AAAA,UACjC,YAAY,KAAK;AAAA,UACjB,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;AAAA,QAC7D;AAAA,MACF,EAAE;AAAA,IACJ,IACA,CAAC;AAAA,IACL,GAAI,QAAQ,iBAAiB,EAAE,gBAAgB,QAAQ,eAAe,IAAI,CAAC;AAAA,EAC7E;AACF;AACA,SAAS,aAAa,SAAiE;AACrF,SAAO,QAAQ,SAAS,IAAI,CAAC,YAAY;AACvC,QAAI,CAAC,CAAC,UAAU,QAAQ,aAAa,MAAM,EAAE,SAAS,QAAQ,IAAI;AAChE,YAAM,IAAI,SAAS,6BAA6B,0CAA0C;AAC5F,QAAI;AACJ,QAAI,QAAQ,OAAO;AACjB,YAAM,OAAO,QAAQ,WAAW;AAChC,UAAI,SAAS;AACb,YAAM,QAA+C,CAAC;AACtD,iBAAW,QAAQ,QAAQ,MAAM,OAAO;AACtC,YAAI,KAAK,SAAS,UAAU,SAAS,KAAK,aAAa,KAAK;AAC1D,gBAAM,IAAI;AAAA,YACR;AAAA,YACA;AAAA,UACF;AACF,cAAM,KAAK,EAAE,MAAM,QAAQ,MAAM,KAAK,MAAM,QAAQ,SAAS,KAAK,UAAU,EAAE,CAAC;AAC/E,kBAAU,KAAK;AAAA,MACjB;AACA,UAAI,WAAW,KAAK;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,UACA;AAAA,QACF;AACF,gBAAU;AAAA,IACZ,WAAW,QAAQ,YAAY,MAAM;AACnC,gBAAU,QAAQ;AAAA,IACpB,OAAO;AACL,YAAM,IAAI,SAAS,6BAA6B,oCAAoC;AAAA,IACtF;AACA,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd;AAAA,MACA,GAAI,QAAQ,SAAS,SAAY,EAAE,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,MAC3D,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC9D,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;AAAA,MACjE,GAAI,QAAQ,eAAe,SAAY,EAAE,YAAY,QAAQ,WAAW,IAAI,CAAC;AAAA,MAC7E,GAAI,QAAQ,iBAAiB,SAAY,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;AAAA,MACnF,GAAI,QAAQ,qBAAqB,SAC7B,EAAE,kBAAkB,QAAQ,iBAAiB,IAC7C,CAAC;AAAA,IACP;AAAA,EACF,CAAC;AACH;AAGO,SAAS,gBAAgB,UAAkC,CAAC,GAAwB;AACzF,SAAO,OAAO,OAAO,YAAY;AAC/B,UAAM,SAAS,UAAU,KAAK;AAC9B,UAAM,UAAU,cAAc,MAAM;AACpC,UAAM,WAAW,aAAa,OAAO;AACrC,UAAM,kBAAkB,YAAY,SAAS,QAAQ,QAAQ,KAAK;AAClE,QAAI,CAAC;AACH,YAAM,IAAI,SAAS,+BAA+B,+BAA+B;AACnF,UAAM,MAAM,QAAQ,SAAS,GAAG,OAAO,QAAQ,IAAI,QAAQ,QAAQ,KAAK;AACxE,QAAI,CAAC,IAAI,SAAS,GAAG;AACnB,YAAM,IAAI,SAAS,+BAA+B,wCAAwC;AAC5F,QAAI;AACJ,QAAI;AACF,iBAAW,QAAQ,gBAAgB,GAAG;AAAA,IACxC,QAAQ;AACN,YAAM,IAAI;AAAA,QACR;AAAA,QACA,kBAAkB,GAAG;AAAA,MACvB;AAAA,IACF;AACA,QAAI,CAAC,QAAQ,SAAS,SAAS,SAAS,SAAS,OAAO;AACtD,YAAM,IAAI;AAAA,QACR;AAAA,QACA,YAAY,GAAG;AAAA,MACjB;AACF,UAAM,cAAc;AAAA,MAClB,GAAG;AAAA,MACH,OAAO,QAAQ,QAAQ,SAAS,QAAQ,QAAQ,QAAQ;AAAA,IAC1D;AACA,QAAI,OAAO,SAAS,QAAQ;AAC1B,YAAM,WAAW,MAAM,SAAS,SAAS,KAAK,UAAU,WAAW;AACnE,aAAO,EAAE,QAAQ,eAAe,UAAU,MAAM,EAAE;AAAA,IACpD;AACA,QAAI,UAAU;AACd,QAAI,WAAW;AAEf,UAAM,QAAQ,oBAAI,IAA6D;AAC/E,QAAI;AACJ,qBAAiB,SAAS,SAAS,SAAS,OAAO,UAAU,WAAW,GAAG;AACzE,UAAI,MAAM,SAAS,aAAc,YAAW,MAAM;AAAA,eACzC,MAAM,SAAS,iBAAkB,aAAY,MAAM;AAAA,eACnD,MAAM,SAAS,mBAAmB;AACzC,cAAM,OAAO,MAAM,IAAI,MAAM,EAAE,KAAK,EAAE,IAAI,MAAM,IAAI,MAAM,IAAI,WAAW,GAAG;AAC5E,YAAI,MAAM,KAAM,MAAK,OAAO,MAAM;AAClC,YAAI,MAAM,UAAW,MAAK,aAAa,MAAM;AAC7C,cAAM,IAAI,MAAM,IAAI,IAAI;AAAA,MAC1B,WAAW,MAAM,SAAS,QAAQ;AAChC,eAAO;AACP;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC;AACH,YAAM,IAAI,SAAS,gCAAgC,oCAAoC;AACzF,WAAO;AAAA,MACL,QAAQ;AAAA,QACN;AAAA,UACE;AAAA,UACA,GAAI,WAAW,EAAE,kBAAkB,SAAS,IAAI,CAAC;AAAA,UACjD,GAAI,MAAM,OACN;AAAA,YACE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE,IAAI,CAAC,UAAU;AAAA,cAC7C,IAAI,KAAK;AAAA,cACT,MAAM;AAAA,cACN,UAAU,EAAE,MAAM,KAAK,MAAM,WAAW,KAAK,UAAU;AAAA,YACzD,EAAE;AAAA,UACJ,IACA,CAAC;AAAA,UACL,GAAI,KAAK,iBAAiB,SAAY,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;AAAA,UAC7E,GAAI,KAAK,oBAAoB,SAAY,EAAE,iBAAiB,KAAK,gBAAgB,IAAI,CAAC;AAAA,UACtF,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,UACvE,GAAI,KAAK,kBAAkB,SAAY,EAAE,eAAe,KAAK,cAAc,IAAI,CAAC;AAAA,QAClF;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC3gBA,SAAS,KAAAC,UAAS;AAGlB,SAAS,aAAa,uBAAuB;AAqC7C,SAAS,eAAe,OAAuC;AAC7D,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,QAAQ;AACd,SACE,MAAM,QAAQ,MAAM,IAAI,KACxB,MAAM,KAAK,MAAM,CAAC,YAAY,OAAO,YAAY,YAAY,OAAO,YAAY,QAAQ,KACxF,OAAO,MAAM,YAAY;AAE7B;AAEO,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAUC,GAAE,OAAO,EAAE,OAAOA,GAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAWA,GAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,SAAS,OAAO;AAAA,IAChB,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AASX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA;AAAA,UACE;AAAA,UACA,aAAa,OAAO,eAAe;AAAA,UACnC,WAAW,OAAO;AAAA,UAClB,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,iBAAiB,OAAO;AAAA,UACxB,MAAM,OAAO;AAAA,UACb,iBAAiB,OAAO;AAAA,UACxB,gBAAgB,EAAE,MAAM,cAAc;AAAA,UACtC,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,8BAAsB,KAAK,YAAY;AAGvC,cAAM,SAAS,OAAO,OAAO,QAAQ,YAAY,YAAY,MAAM,IAAI,SAAS;AAChF,YAAI,MAAM,QAAQ,MAAM,KAAK,OAAO,MAAM,cAAc,GAAG;AACzD,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,gCAAsB,OAAO,YAAY;AACzC,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACtJO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":["path","readFile","readFile","z","z"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.25.2",
+  "version": "0.26.0",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.25.2"
+    "@axlsdk/axl": "0.26.0"
   },
   "peerDependencies": {
     "zod": "^4.0.0",