@sembl/testing 0.2.0 → 0.3.0

package/README.md

@@ -69,7 +69,11 @@ provider:
 export { Listing as schema } from "./dist/schemas.js";
 export const provider = new AnthropicProvider({ model: "claude-sonnet-5", apiKey: process.env.ANTHROPIC_API_KEY });
 export const prices = { inputPerMTok: 3, outputPerMTok: 15 };
-export const coerceOptions = { onInvalidField: "clamp", maxInputChars: 40_000 };
+export const coerceOptions = {
+  onInvalidField: "clamp",
+  maxInputChars: 40_000,
+  instructions: ["Guest counts exclude infants."],
+};
 ```
 
 ```sh

package/dist/index.cjs

@@ -459,11 +459,12 @@ function formatReport(report, diff) {
   const width = Math.max(5, ...report.fields.map((f) => f.path.length));
   const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));
   lines.push(`${"Field".padEnd(width)}  Prec  Recall   tp  fp  fn  \u0394`);
+  const col = (value, w) => pct(value).trim().padStart(w);
   for (const field of report.fields) {
     const d = deltas.get(field.path);
     const change = d ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : "", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : ""].filter(Boolean).join(" ") : "";
     lines.push(
-      `${field.path.padEnd(width)}  ${pct(field.precision)} ${pct(field.recall)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`
+      `${field.path.padEnd(width)}  ${col(field.precision, 4)}  ${col(field.recall, 6)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`
     );
   }
   lines.push("");

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/replay.ts","../src/eval.ts"],"sourcesContent":["export {\n  RecordingProvider,\n  ReplayProvider,\n  ReplayMissError,\n  replayOrRecord,\n  recordingKey,\n  recordingPath,\n} from \"./replay.js\";\nexport type { Recording, ReplayOptions } from \"./replay.js\";\n\nexport {\n  runEval,\n  loadFixtures,\n  compareLeaves,\n  flattenLeaves,\n  leavesEqual,\n  fieldStats,\n  estimateCost,\n  saveReport,\n  loadReport,\n  diffReports,\n  formatReport,\n} from \"./eval.js\";\nexport type {\n  EvalFixture,\n  EvalOptions,\n  EvalReport,\n  EvalItem,\n  EvalTotals,\n  EvalDiff,\n  FieldDelta,\n  FieldStats,\n  LeafResult,\n  LeafOutcome,\n  TokenPrices,\n  Usage,\n} from \"./eval.js\";\n","import { createHash } from \"node:crypto\";\nimport { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Provider, ProviderRequest, ProviderResponse } from \"@sembl/core\";\n\n/** What a recording holds: enough of the request to recognise it, and the answer. */\nexport interface Recording {\n  /** Hash of the parts of the request that determine the answer. */\n  key: string;\n  schemaId: string;\n  request: {\n    systemPrompt: string;\n    userInput: string;\n    jsonSchema: Record<string, unknown>;\n  };\n  response: ProviderResponse;\n  recordedAt: string;\n}\n\n/** JSON with sorted keys, so the same request always hashes the same. */\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_key, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\n/**\n * The key a request records under. Only the parts that reach the model\n * count: the system prompt, the user input and the JSON Schema. The runtime\n * schema and bundle are already folded into those.\n */\nexport function recordingKey(request: ProviderRequest): string {\n  const material = stable({\n    systemPrompt: request.systemPrompt,\n    userInput: request.userInput,\n    jsonSchema: request.jsonSchema,\n  });\n  return createHash(\"sha256\").update(material).digest(\"hex\").slice(0, 24);\n}\n\nfunction slug(id: string): string {\n  return id.replace(/[^A-Za-z0-9_-]+/g, \"_\").slice(0, 40) || \"schema\";\n}\n\n/** Where a recording for a request lives inside a directory. */\nexport function recordingPath(dir: string, request: ProviderRequest): string {\n  return join(dir, `${slug(request.schema.id)}.${recordingKey(request)}.json`);\n}\n\n/** Thrown by {@link ReplayProvider} when no recording matches a request. */\nexport class ReplayMissError extends Error {\n  constructor(\n    public readonly key: string,\n    public readonly dir: string,\n    public readonly schemaId: string,\n  ) {\n    super(\n      `No recording for a \"${schemaId}\" request (key ${key}) in ${dir}. ` +\n        \"Run once with a RecordingProvider, or pass a fallback provider to record misses.\",\n    );\n    this.name = \"ReplayMissError\";\n  }\n}\n\n/**\n * Calls another provider and writes every request/response pair to a\n * directory, one JSON file each, named by schema and request hash. Run your\n * extraction code through it once with real credentials; the files it leaves\n * behind let a {@link ReplayProvider} answer the same requests offline.\n *\n * A request is identified by what reaches the model, so editing a field\n * description or the input produces a new recording rather than a stale hit.\n */\nexport class RecordingProvider implements Provider {\n  constructor(\n    private readonly inner: Provider,\n    private readonly dir: string,\n  ) {\n    mkdirSync(dir, { recursive: true });\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const recording: Recording = {\n      key: recordingKey(request),\n      schemaId: request.schema.id,\n      request: {\n        systemPrompt: request.systemPrompt,\n        userInput: request.userInput,\n        jsonSchema: request.jsonSchema,\n      },\n      response,\n      recordedAt: new Date().toISOString(),\n    };\n    writeFileSync(recordingPath(this.dir, request), JSON.stringify(recording, null, 2) + \"\\n\");\n    return response;\n  }\n}\n\n/** Options for {@link ReplayProvider}. */\nexport interface ReplayOptions {\n  /**\n   * Where to send a request no recording matches. The answer is recorded,\n   * so the next run replays it. Without one, a miss throws\n   * {@link ReplayMissError} — the right behaviour in CI, where a miss means\n   * a fixture changed and nobody re-recorded.\n   */\n  fallback?: Provider;\n}\n\n/**\n * Answers requests from a directory of recordings, never touching the\n * network. Deterministic, free, and fast: the provider to put under tests of\n * your own extraction code.\n */\nexport class ReplayProvider implements Provider {\n  private readonly recorder: RecordingProvider | undefined;\n\n  constructor(\n    private readonly dir: string,\n    options: ReplayOptions = {},\n  ) {\n    this.recorder = options.fallback ? new RecordingProvider(options.fallback, dir) : undefined;\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const path = recordingPath(this.dir, request);\n    if (existsSync(path)) {\n      const recording = JSON.parse(readFileSync(path, \"utf8\")) as Recording;\n      return recording.response;\n    }\n    if (this.recorder) {\n      return this.recorder.complete(request);\n    }\n    throw new ReplayMissError(recordingKey(request), this.dir, request.schema.id);\n  }\n\n  /** How many recordings the directory holds. */\n  size(): number {\n    if (!existsSync(this.dir)) return 0;\n    return readdirSync(this.dir).filter((f) => f.endsWith(\".json\")).length;\n  }\n}\n\n/**\n * The usual arrangement: replay from `dir`, and record misses through\n * `live` when it is given — say, only when an API key is present — so the\n * same test file works locally with credentials and in CI without them.\n */\nexport function replayOrRecord(dir: string, live?: Provider): Provider {\n  return new ReplayProvider(dir, live ? { fallback: live } : {});\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\";\nimport { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from \"node:fs\";\nimport { basename, dirname, join, resolve } from \"node:path\";\nimport {\n  coerce,\n  partialCoerce,\n  coerceWithProvenance,\n  partialCoerceWithProvenance,\n} from \"@sembl/core\";\nimport type {\n  CoerceInput,\n  CoerceOptions,\n  FieldProvenance,\n  Provider,\n  ProviderRequest,\n  ProviderResponse,\n  ProviderUsage,\n  ResolvedIssue,\n  Source,\n} from \"@sembl/core\";\n\n/** One case: an input and what a correct extraction of it looks like. */\nexport interface EvalFixture {\n  /** Shown in the report. Defaults to the file it was loaded from. */\n  name?: string;\n  input: CoerceInput;\n  /** The expected extraction. Absent and `null` both mean \"not present\". */\n  expected: Record<string, unknown>;\n}\n\n/** Per-million-token prices, for a cost line in the report. */\nexport interface TokenPrices {\n  inputPerMTok: number;\n  outputPerMTok: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache reads. */\n  cacheReadPerMTok?: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache writes. */\n  cacheWritePerMTok?: number;\n}\n\n/** Options for {@link runEval}. Everything in `CoerceOptions` applies per fixture. */\nexport interface EvalOptions extends CoerceOptions {\n  fixtures: readonly EvalFixture[];\n  /** Which coercion to run. Default `\"coerce\"`. */\n  mode?: \"coerce\" | \"partialCoerce\";\n  /** Ask for provenance, so the report can show confidence per field. */\n  provenance?: boolean;\n  /** How many fixtures run at once. Default 1, for stable latency numbers. */\n  concurrency?: number;\n  /** Prices for the cost line. Without them the report has no cost. */\n  prices?: TokenPrices;\n}\n\n/** How one leaf compared. */\nexport type LeafOutcome = \"match\" | \"wrong\" | \"missing\" | \"extra\";\n\nexport interface LeafResult {\n  path: string;\n  outcome: LeafOutcome;\n  expected?: unknown;\n  actual?: unknown;\n  confidence?: FieldProvenance[\"confidence\"];\n}\n\nexport interface EvalItem {\n  name: string;\n  ok: boolean;\n  /** Message of the error the coercion threw, when it did. */\n  error?: string;\n  /** Whether every expected leaf matched and nothing extra came back. */\n  exact: boolean;\n  leaves: LeafResult[];\n  issues: ResolvedIssue[];\n  latencyMs: number;\n  calls: number;\n  usage: Usage;\n}\n\nexport interface Usage {\n  promptTokens: number;\n  completionTokens: number;\n  cacheReadTokens: number;\n  cacheWriteTokens: number;\n}\n\nexport interface FieldStats {\n  path: string;\n  tp: number;\n  fp: number;\n  fn: number;\n  /** `tp / (tp + fp)`; null when nothing was returned for the field. */\n  precision: number | null;\n  /** `tp / (tp + fn)`; null when nothing was expected for the field. */\n  recall: number | null;\n}\n\nexport interface EvalTotals {\n  fixtures: number;\n  ok: number;\n  exact: number;\n  precision: number | null;\n  recall: number | null;\n  usage: Usage;\n  calls: number;\n  cost?: number;\n  latencyMs: { p50: number; p95: number; max: number; total: number };\n}\n\nexport interface EvalReport {\n  schemaId: string;\n  mode: \"coerce\" | \"partialCoerce\";\n  ranAt: string;\n  items: EvalItem[];\n  fields: FieldStats[];\n  totals: EvalTotals;\n}\n\n// ---------------------------------------------------------------------------\n// Fixtures\n\ninterface FileInput {\n  file: string;\n  label?: string;\n}\n\nfunction isFileInput(value: unknown): value is FileInput {\n  return typeof value === \"object\" && value !== null && typeof (value as FileInput).file === \"string\";\n}\n\n/** Resolve `{ file, label }` inputs against the fixture's own directory. */\nfunction resolveInput(input: unknown, baseDir: string): CoerceInput {\n  const one = (value: unknown): Source | string => {\n    if (typeof value === \"string\") return value;\n    if (isFileInput(value)) {\n      const text = readFileSync(resolve(baseDir, value.file), \"utf8\");\n      return value.label ? { label: value.label, text } : { text };\n    }\n    return value as Source;\n  };\n  if (Array.isArray(input)) {\n    return input.map((v) => {\n      const r = one(v);\n      return typeof r === \"string\" ? { text: r } : r;\n    });\n  }\n  return one(input);\n}\n\n/**\n * Load fixtures from a directory: every `*.json` file is either one fixture\n * or an array of them. An input may be `{ \"file\": \"page.html\", \"label\": … }`\n * to pull text from a sibling file, which keeps large scraped pages out of\n * the JSON.\n */\nexport function loadFixtures(dir: string): EvalFixture[] {\n  const root = resolve(dir);\n  if (!existsSync(root)) {\n    throw new Error(`Fixture directory not found: ${root}`);\n  }\n  const fixtures: EvalFixture[] = [];\n  const files = readdirSync(root)\n    .filter((f) => f.endsWith(\".json\") && !f.startsWith(\".\"))\n    .sort();\n  for (const file of files) {\n    const path = join(root, file);\n    const parsed = JSON.parse(readFileSync(path, \"utf8\")) as unknown;\n    const list = Array.isArray(parsed) ? parsed : [parsed];\n    list.forEach((raw, index) => {\n      const fixture = raw as EvalFixture;\n      if (!fixture || typeof fixture !== \"object\" || !(\"input\" in fixture) || !(\"expected\" in fixture)) {\n        throw new Error(`${path}${list.length > 1 ? `[${index}]` : \"\"}: a fixture needs \"input\" and \"expected\"`);\n      }\n      const stem = basename(file, \".json\");\n      fixtures.push({\n        name: fixture.name ?? (list.length > 1 ? `${stem}[${index}]` : stem),\n        input: resolveInput(fixture.input, dirname(path)),\n        expected: fixture.expected,\n      });\n    });\n  }\n  return fixtures;\n}\n\n// ---------------------------------------------------------------------------\n// Scoring\n\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_k, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\nfunction isPrimitive(value: unknown): boolean {\n  return value === null || typeof value !== \"object\";\n}\n\n/**\n * Flatten a value to its leaves, keyed by dotted path. Objects recurse;\n * arrays are leaves. `null` and `undefined` are absence, not leaves.\n */\nexport function flattenLeaves(value: unknown, prefix = \"\"): Map<string, unknown> {\n  const leaves = new Map<string, unknown>();\n  if (value === null || value === undefined) return leaves;\n  if (typeof value !== \"object\" || Array.isArray(value)) {\n    leaves.set(prefix, value);\n    return leaves;\n  }\n  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {\n    const path = prefix ? `${prefix}.${key}` : key;\n    for (const [leafPath, leaf] of flattenLeaves(child, path)) {\n      leaves.set(leafPath, leaf);\n    }\n  }\n  return leaves;\n}\n\n/**\n * Whether two leaves agree. Arrays of primitives compare as multisets —\n * the order amenities come back in is not a fact about the listing —\n * and everything else compares structurally.\n */\nexport function leavesEqual(a: unknown, b: unknown): boolean {\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    if (a.every(isPrimitive) && b.every(isPrimitive)) {\n      const sortedA = [...a].map(String).sort();\n      const sortedB = [...b].map(String).sort();\n      return sortedA.every((v, i) => v === sortedB[i]);\n    }\n  }\n  return stable(a) === stable(b);\n}\n\n/** Compare one extraction against its expectation, leaf by leaf. */\nexport function compareLeaves(\n  expected: Record<string, unknown>,\n  actual: Record<string, unknown> | undefined,\n  provenance: Record<string, FieldProvenance> = {},\n): LeafResult[] {\n  const want = flattenLeaves(expected);\n  const got = flattenLeaves(actual ?? {});\n  const paths = [...new Set([...want.keys(), ...got.keys()])].sort();\n  return paths.map((path) => {\n    const top = path.split(\".\")[0];\n    const confidence = provenance[top]?.confidence;\n    const result: LeafResult = { path, outcome: \"match\" };\n    if (confidence) result.confidence = confidence;\n    if (want.has(path) && got.has(path)) {\n      result.expected = want.get(path);\n      result.actual = got.get(path);\n      result.outcome = leavesEqual(want.get(path), got.get(path)) ? \"match\" : \"wrong\";\n    } else if (want.has(path)) {\n      result.expected = want.get(path);\n      result.outcome = \"missing\";\n    } else {\n      result.actual = got.get(path);\n      result.outcome = \"extra\";\n    }\n    return result;\n  });\n}\n\nfunction ratio(num: number, den: number): number | null {\n  return den === 0 ? null : num / den;\n}\n\n/** Aggregate leaf outcomes into per-field precision and recall. */\nexport function fieldStats(items: readonly EvalItem[]): FieldStats[] {\n  const byPath = new Map<string, { tp: number; fp: number; fn: number }>();\n  for (const item of items) {\n    for (const leaf of item.leaves) {\n      const stats = byPath.get(leaf.path) ?? { tp: 0, fp: 0, fn: 0 };\n      switch (leaf.outcome) {\n        case \"match\":\n          stats.tp += 1;\n          break;\n        case \"wrong\":\n          stats.fp += 1;\n          stats.fn += 1;\n          break;\n        case \"missing\":\n          stats.fn += 1;\n          break;\n        case \"extra\":\n          stats.fp += 1;\n          break;\n      }\n      byPath.set(leaf.path, stats);\n    }\n  }\n  return [...byPath.entries()]\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([path, { tp, fp, fn }]) => ({\n      path,\n      tp,\n      fp,\n      fn,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n    }));\n}\n\nfunction percentile(sorted: readonly number[], p: number): number {\n  if (sorted.length === 0) return 0;\n  const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);\n  return sorted[Math.max(0, index)];\n}\n\nfunction emptyUsage(): Usage {\n  return { promptTokens: 0, completionTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };\n}\n\nfunction addUsage(into: Usage, usage: ProviderUsage | undefined): void {\n  if (!usage) return;\n  into.promptTokens += usage.promptTokens;\n  into.completionTokens += usage.completionTokens;\n  into.cacheReadTokens += usage.cacheReadTokens ?? 0;\n  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;\n}\n\n/** Dollars, from per-million-token prices. */\nexport function estimateCost(usage: Usage, prices: TokenPrices): number {\n  const per = (tokens: number, price: number) => (tokens / 1_000_000) * price;\n  return (\n    per(usage.promptTokens, prices.inputPerMTok) +\n    per(usage.completionTokens, prices.outputPerMTok) +\n    per(usage.cacheReadTokens, prices.cacheReadPerMTok ?? prices.inputPerMTok) +\n    per(usage.cacheWriteTokens, prices.cacheWritePerMTok ?? prices.inputPerMTok)\n  );\n}\n\n// ---------------------------------------------------------------------------\n// Running\n\ninterface ItemContext {\n  usage: Usage;\n  calls: number;\n}\n\nconst context = new AsyncLocalStorage<ItemContext>();\n\n/** Credits each call's usage to whichever fixture is running it. */\nclass MeteredProvider implements Provider {\n  constructor(private readonly inner: Provider) {}\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const ctx = context.getStore();\n    if (ctx) {\n      ctx.calls += 1;\n      addUsage(ctx.usage, response.usage);\n    }\n    return response;\n  }\n}\n\nasync function mapWithConcurrency<T, R>(\n  items: readonly T[],\n  concurrency: number,\n  fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results: R[] = new Array(items.length);\n  let next = 0;\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {\n    while (next < items.length) {\n      const index = next++;\n      results[index] = await fn(items[index], index);\n    }\n  });\n  await Promise.all(workers);\n  return results;\n}\n\n/**\n * Run every fixture through a coercion and score the results.\n *\n * Per-field precision and recall make a description change measurable: a\n * field whose recall dropped after a rewording is a field the rewording\n * hurt. Token usage and latency come along so the cost of a change is\n * visible next to its effect. Pair it with a `ReplayProvider` to run in CI\n * without spend.\n */\nexport async function runEval(options: EvalOptions): Promise<EvalReport> {\n  const {\n    fixtures,\n    mode = \"coerce\",\n    provenance = false,\n    concurrency = 1,\n    prices,\n    provider,\n    ...coerceOptions\n  } = options;\n  const metered = new MeteredProvider(provider);\n  const run = mode === \"coerce\"\n    ? provenance ? coerceWithProvenance : coerce\n    : provenance ? partialCoerceWithProvenance : partialCoerce;\n\n  const items = await mapWithConcurrency(fixtures, concurrency, async (fixture, index) => {\n    const ctx: ItemContext = { usage: emptyUsage(), calls: 0 };\n    const name = fixture.name ?? `fixture ${index + 1}`;\n    const started = performance.now();\n    return context.run(ctx, async (): Promise<EvalItem> => {\n      try {\n        const result = await run<Record<string, unknown>>(fixture.input, {\n          ...coerceOptions,\n          provider: metered,\n        });\n        const data = provenance\n          ? (result as { data: Record<string, unknown> }).data\n          : (result as Record<string, unknown>);\n        const prov = provenance ? (result as { provenance: Record<string, FieldProvenance> }).provenance : {};\n        const issues = provenance ? (result as { issues: ResolvedIssue[] }).issues : [];\n        const leaves = compareLeaves(fixture.expected, data, prov);\n        return {\n          name,\n          ok: true,\n          exact: leaves.every((l) => l.outcome === \"match\"),\n          leaves,\n          issues,\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      } catch (error) {\n        return {\n          name,\n          ok: false,\n          error: error instanceof Error ? error.message : String(error),\n          exact: false,\n          leaves: compareLeaves(fixture.expected, undefined),\n          issues: [],\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      }\n    });\n  });\n\n  const fields = fieldStats(items);\n  const tp = fields.reduce((s, f) => s + f.tp, 0);\n  const fp = fields.reduce((s, f) => s + f.fp, 0);\n  const fn = fields.reduce((s, f) => s + f.fn, 0);\n  const usage = emptyUsage();\n  for (const item of items) {\n    usage.promptTokens += item.usage.promptTokens;\n    usage.completionTokens += item.usage.completionTokens;\n    usage.cacheReadTokens += item.usage.cacheReadTokens;\n    usage.cacheWriteTokens += item.usage.cacheWriteTokens;\n  }\n  const latencies = items.map((i) => i.latencyMs).sort((a, b) => a - b);\n\n  return {\n    schemaId: options.schema.id,\n    mode,\n    ranAt: new Date().toISOString(),\n    items,\n    fields,\n    totals: {\n      fixtures: items.length,\n      ok: items.filter((i) => i.ok).length,\n      exact: items.filter((i) => i.exact).length,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n      usage,\n      calls: items.reduce((s, i) => s + i.calls, 0),\n      ...(prices ? { cost: estimateCost(usage, prices) } : {}),\n      latencyMs: {\n        p50: percentile(latencies, 50),\n        p95: percentile(latencies, 95),\n        max: latencies[latencies.length - 1] ?? 0,\n        total: latencies.reduce((s, l) => s + l, 0),\n      },\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Persistence and diffs\n\nexport function saveReport(report: EvalReport, file: string): void {\n  mkdirSync(dirname(resolve(file)), { recursive: true });\n  writeFileSync(resolve(file), JSON.stringify(report, null, 2) + \"\\n\");\n}\n\nexport function loadReport(file: string): EvalReport | undefined {\n  const path = resolve(file);\n  if (!existsSync(path)) return undefined;\n  return JSON.parse(readFileSync(path, \"utf8\")) as EvalReport;\n}\n\nexport interface FieldDelta {\n  path: string;\n  precision: number | null;\n  recall: number | null;\n}\n\n/** What changed between two runs. Deltas are `next − previous`. */\nexport interface EvalDiff {\n  exact: number;\n  ok: number;\n  precision: number | null;\n  recall: number | null;\n  promptTokens: number;\n  completionTokens: number;\n  cost?: number;\n  p50Ms: number;\n  /** Fields whose precision or recall moved, largest drop first. */\n  fields: FieldDelta[];\n}\n\nfunction delta(a: number | null | undefined, b: number | null | undefined): number | null {\n  return a === null || a === undefined || b === null || b === undefined ? null : b - a;\n}\n\nexport function diffReports(previous: EvalReport, next: EvalReport): EvalDiff {\n  const prevFields = new Map(previous.fields.map((f) => [f.path, f]));\n  const fields: FieldDelta[] = [];\n  for (const field of next.fields) {\n    const before = prevFields.get(field.path);\n    const precision = delta(before?.precision, field.precision);\n    const recall = delta(before?.recall, field.recall);\n    if ((precision !== null && precision !== 0) || (recall !== null && recall !== 0)) {\n      fields.push({ path: field.path, precision, recall });\n    }\n  }\n  fields.sort((a, b) => Math.min(a.precision ?? 0, a.recall ?? 0) - Math.min(b.precision ?? 0, b.recall ?? 0));\n\n  return {\n    exact: next.totals.exact - previous.totals.exact,\n    ok: next.totals.ok - previous.totals.ok,\n    precision: delta(previous.totals.precision, next.totals.precision),\n    recall: delta(previous.totals.recall, next.totals.recall),\n    promptTokens: next.totals.usage.promptTokens - previous.totals.usage.promptTokens,\n    completionTokens: next.totals.usage.completionTokens - previous.totals.usage.completionTokens,\n    ...(next.totals.cost !== undefined && previous.totals.cost !== undefined\n      ? { cost: next.totals.cost - previous.totals.cost }\n      : {}),\n    p50Ms: next.totals.latencyMs.p50 - previous.totals.latencyMs.p50,\n    fields,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Formatting\n\nfunction pct(value: number | null): string {\n  return value === null ? \"  –  \" : `${(value * 100).toFixed(0).padStart(3)}%`;\n}\n\nfunction signed(value: number | null, digits = 0, suffix = \"\"): string {\n  if (value === null || value === 0) return \"\";\n  const text = digits > 0 ? value.toFixed(digits) : String(value);\n  return ` (${value > 0 ? \"+\" : \"\"}${text}${suffix})`;\n}\n\nfunction signedPct(value: number | null): string {\n  return value === null || value === 0 ? \"\" : ` (${value > 0 ? \"+\" : \"\"}${(value * 100).toFixed(0)}pt)`;\n}\n\n/** A plain-text report for a terminal, with deltas when a previous run is given. */\nexport function formatReport(report: EvalReport, diff?: EvalDiff): string {\n  const t = report.totals;\n  const lines: string[] = [];\n  lines.push(\n    `Eval: ${report.schemaId} (${report.mode}) — ${t.fixtures} fixture(s), ${t.ok} ran${signed(diff?.ok ?? null)}, ${t.exact} exact${signed(diff?.exact ?? null)}`,\n  );\n  lines.push(\"\");\n\n  const width = Math.max(5, ...report.fields.map((f) => f.path.length));\n  const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));\n  lines.push(`${\"Field\".padEnd(width)}  Prec  Recall   tp  fp  fn  Δ`);\n  for (const field of report.fields) {\n    const d = deltas.get(field.path);\n    const change = d\n      ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : \"\", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : \"\"]\n          .filter(Boolean)\n          .join(\" \")\n      : \"\";\n    lines.push(\n      `${field.path.padEnd(width)}  ${pct(field.precision)} ${pct(field.recall)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`,\n    );\n  }\n  lines.push(\"\");\n  lines.push(\n    `Overall: precision ${pct(t.precision).trim()}${signedPct(diff?.precision ?? null)}, recall ${pct(t.recall).trim()}${signedPct(diff?.recall ?? null)}`,\n  );\n  const u = t.usage;\n  const cache = u.cacheReadTokens || u.cacheWriteTokens\n    ? `, cache read ${u.cacheReadTokens.toLocaleString(\"en-US\")} / write ${u.cacheWriteTokens.toLocaleString(\"en-US\")}`\n    : \"\";\n  lines.push(\n    `Tokens: ${u.promptTokens.toLocaleString(\"en-US\")} prompt${signed(diff?.promptTokens ?? null)} / ${u.completionTokens.toLocaleString(\"en-US\")} completion${signed(diff?.completionTokens ?? null)}${cache}; ${t.calls} call(s)` +\n      (t.cost !== undefined ? `; cost $${t.cost.toFixed(4)}${signed(diff?.cost ?? null, 4)}` : \"\"),\n  );\n  lines.push(\n    `Latency: p50 ${t.latencyMs.p50}ms${signed(diff?.p50Ms ?? null, 0, \"ms\")}, p95 ${t.latencyMs.p95}ms, max ${t.latencyMs.max}ms`,\n  );\n\n  const failed = report.items.filter((i) => !i.ok);\n  if (failed.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Failed:\");\n    for (const item of failed) lines.push(`  ${item.name}: ${item.error}`);\n  }\n  const imperfect = report.items.filter((i) => i.ok && !i.exact);\n  if (imperfect.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Mismatches:\");\n    for (const item of imperfect) {\n      for (const leaf of item.leaves.filter((l) => l.outcome !== \"match\")) {\n        const detail =\n          leaf.outcome === \"wrong\"\n            ? `expected ${JSON.stringify(leaf.expected)}, got ${JSON.stringify(leaf.actual)}`\n            : leaf.outcome === \"missing\"\n              ? `expected ${JSON.stringify(leaf.expected)}, got nothing`\n              : `unexpected ${JSON.stringify(leaf.actual)}`;\n        const conf = leaf.confidence ? ` [${leaf.confidence}]` : \"\";\n        lines.push(`  ${item.name} › ${leaf.path}: ${detail}${conf}`);\n      }\n    }\n  }\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,yBAA2B;AAC3B,qBAAgF;AAChF,uBAAqB;AAkBrB,SAAS,OAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,MAAM,MAClC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAOO,SAAS,aAAa,SAAkC;AAC7D,QAAM,WAAW,OAAO;AAAA,IACtB,cAAc,QAAQ;AAAA,IACtB,WAAW,QAAQ;AAAA,IACnB,YAAY,QAAQ;AAAA,EACtB,CAAC;AACD,aAAO,+BAAW,QAAQ,EAAE,OAAO,QAAQ,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AACxE;AAEA,SAAS,KAAK,IAAoB;AAChC,SAAO,GAAG,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK;AAC7D;AAGO,SAAS,cAAc,KAAa,SAAkC;AAC3E,aAAO,uBAAK,KAAK,GAAG,KAAK,QAAQ,OAAO,EAAE,CAAC,IAAI,aAAa,OAAO,CAAC,OAAO;AAC7E;AAGO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EACzC,YACkB,KACA,KACA,UAChB;AACA;AAAA,MACE,uBAAuB,QAAQ,kBAAkB,GAAG,QAAQ,GAAG;AAAA,IAEjE;AAPgB;AACA;AACA;AAMhB,SAAK,OAAO;AAAA,EACd;AACF;AAWO,IAAM,oBAAN,MAA4C;AAAA,EACjD,YACmB,OACA,KACjB;AAFiB;AACA;AAEjB,kCAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,YAAuB;AAAA,MAC3B,KAAK,aAAa,OAAO;AAAA,MACzB,UAAU,QAAQ,OAAO;AAAA,MACzB,SAAS;AAAA,QACP,cAAc,QAAQ;AAAA,QACtB,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,MACA,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,sCAAc,cAAc,KAAK,KAAK,OAAO,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,IAAI,IAAI;AACzF,WAAO;AAAA,EACT;AACF;AAkBO,IAAM,iBAAN,MAAyC;AAAA,EAG9C,YACmB,KACjB,UAAyB,CAAC,GAC1B;AAFiB;AAGjB,SAAK,WAAW,QAAQ,WAAW,IAAI,kBAAkB,QAAQ,UAAU,GAAG,IAAI;AAAA,EACpF;AAAA,EAPiB;AAAA,EASjB,MAAM,SAAS,SAAqD;AAClE,UAAM,OAAO,cAAc,KAAK,KAAK,OAAO;AAC5C,YAAI,2BAAW,IAAI,GAAG;AACpB,YAAM,YAAY,KAAK,UAAM,6BAAa,MAAM,MAAM,CAAC;AACvD,aAAO,UAAU;AAAA,IACnB;AACA,QAAI,KAAK,UAAU;AACjB,aAAO,KAAK,SAAS,SAAS,OAAO;AAAA,IACvC;AACA,UAAM,IAAI,gBAAgB,aAAa,OAAO,GAAG,KAAK,KAAK,QAAQ,OAAO,EAAE;AAAA,EAC9E;AAAA;AAAA,EAGA,OAAe;AACb,QAAI,KAAC,2BAAW,KAAK,GAAG,EAAG,QAAO;AAClC,eAAO,4BAAY,KAAK,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,EAAE;AAAA,EAClE;AACF;AAOO,SAAS,eAAe,KAAa,MAA2B;AACrE,SAAO,IAAI,eAAe,KAAK,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC,CAAC;AAC/D;;;ACzJA,8BAAkC;AAClC,IAAAA,kBAAgF;AAChF,IAAAC,oBAAiD;AACjD,kBAKO;AAqHP,SAAS,YAAY,OAAoC;AACvD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAoB,SAAS;AAC7F;AAGA,SAAS,aAAa,OAAgB,SAA8B;AAClE,QAAM,MAAM,CAAC,UAAoC;AAC/C,QAAI,OAAO,UAAU,SAAU,QAAO;AACtC,QAAI,YAAY,KAAK,GAAG;AACtB,YAAM,WAAO,kCAAa,2BAAQ,SAAS,MAAM,IAAI,GAAG,MAAM;AAC9D,aAAO,MAAM,QAAQ,EAAE,OAAO,MAAM,OAAO,KAAK,IAAI,EAAE,KAAK;AAAA,IAC7D;AACA,WAAO;AAAA,EACT;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM,IAAI,CAAC,MAAM;AACtB,YAAM,IAAI,IAAI,CAAC;AACf,aAAO,OAAO,MAAM,WAAW,EAAE,MAAM,EAAE,IAAI;AAAA,IAC/C,CAAC;AAAA,EACH;AACA,SAAO,IAAI,KAAK;AAClB;AAQO,SAAS,aAAa,KAA4B;AACvD,QAAM,WAAO,2BAAQ,GAAG;AACxB,MAAI,KAAC,4BAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,gCAAgC,IAAI,EAAE;AAAA,EACxD;AACA,QAAM,WAA0B,CAAC;AACjC,QAAM,YAAQ,6BAAY,IAAI,EAC3B,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC,EACvD,KAAK;AACR,aAAW,QAAQ,OAAO;AACxB,UAAM,WAAO,wBAAK,MAAM,IAAI;AAC5B,UAAM,SAAS,KAAK,UAAM,8BAAa,MAAM,MAAM,CAAC;AACpD,UAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AACrD,SAAK,QAAQ,CAAC,KAAK,UAAU;AAC3B,YAAM,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,YAAY,YAAY,EAAE,WAAW,YAAY,EAAE,cAAc,UAAU;AAChG,cAAM,IAAI,MAAM,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,EAAE,0CAA0C;AAAA,MACzG;AACA,YAAM,WAAO,4BAAS,MAAM,OAAO;AACnC,eAAS,KAAK;AAAA,QACZ,MAAM,QAAQ,SAAS,KAAK,SAAS,IAAI,GAAG,IAAI,IAAI,KAAK,MAAM;AAAA,QAC/D,OAAO,aAAa,QAAQ,WAAO,2BAAQ,IAAI,CAAC;AAAA,QAChD,UAAU,QAAQ;AAAA,MACpB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAKA,SAASC,QAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,IAAI,MAChC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAEA,SAAS,YAAY,OAAyB;AAC5C,SAAO,UAAU,QAAQ,OAAO,UAAU;AAC5C;AAMO,SAAS,cAAc,OAAgB,SAAS,IAA0B;AAC/E,QAAM,SAAS,oBAAI,IAAqB;AACxC,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACrD,WAAO,IAAI,QAAQ,KAAK;AACxB,WAAO;AAAA,EACT;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAgC,GAAG;AAC3E,UAAM,OAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,eAAW,CAAC,UAAU,IAAI,KAAK,cAAc,OAAO,IAAI,GAAG;AACzD,aAAO,IAAI,UAAU,IAAI;AAAA,IAC3B;AAAA,EACF;AACA,SAAO;AACT;AAOO,SAAS,YAAY,GAAY,GAAqB;AAC3D,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,QAAI,EAAE,MAAM,WAAW,KAAK,EAAE,MAAM,WAAW,GAAG;AAChD,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,aAAO,QAAQ,MAAM,CAAC,GAAG,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,IACjD;AAAA,EACF;AACA,SAAOA,QAAO,CAAC,MAAMA,QAAO,CAAC;AAC/B;AAGO,SAAS,cACd,UACA,QACA,aAA8C,CAAC,GACjC;AACd,QAAM,OAAO,cAAc,QAAQ;AACnC,QAAM,MAAM,cAAc,UAAU,CAAC,CAAC;AACtC,QAAM,QAAQ,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,KAAK,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACjE,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,MAAM,KAAK,MAAM,GAAG,EAAE,CAAC;AAC7B,UAAM,aAAa,WAAW,GAAG,GAAG;AACpC,UAAM,SAAqB,EAAE,MAAM,SAAS,QAAQ;AACpD,QAAI,WAAY,QAAO,aAAa;AACpC,QAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG;AACnC,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU,YAAY,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,UAAU;AAAA,IAC1E,WAAW,KAAK,IAAI,IAAI,GAAG;AACzB,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,UAAU;AAAA,IACnB,OAAO;AACL,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU;AAAA,IACnB;AACA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,MAAM,KAAa,KAA4B;AACtD,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAGO,SAAS,WAAW,OAA0C;AACnE,QAAM,SAAS,oBAAI,IAAoD;AACvE,aAAW,QAAQ,OAAO;AACxB,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,QAAQ,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE;AAC7D,cAAQ,KAAK,SAAS;AAAA,QACpB,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,MACJ;AACA,aAAO,IAAI,KAAK,MAAM,KAAK;AAAA,IAC7B;AAAA,EACF;AACA,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EACxB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO;AAAA,IAChC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,IAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,EAC3B,EAAE;AACN;AAEA,SAAS,WAAW,QAA2B,GAAmB;AAChE,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,QAAQ,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,IAAI,CAAC;AAClF,SAAO,OAAO,KAAK,IAAI,GAAG,KAAK,CAAC;AAClC;AAEA,SAAS,aAAoB;AAC3B,SAAO,EAAE,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,kBAAkB,EAAE;AACzF;AAEA,SAAS,SAAS,MAAa,OAAwC;AACrE,MAAI,CAAC,MAAO;AACZ,OAAK,gBAAgB,MAAM;AAC3B,OAAK,oBAAoB,MAAM;AAC/B,OAAK,mBAAmB,MAAM,mBAAmB;AACjD,OAAK,oBAAoB,MAAM,oBAAoB;AACrD;AAGO,SAAS,aAAa,OAAc,QAA6B;AACtE,QAAM,MAAM,CAAC,QAAgB,UAAmB,SAAS,MAAa;AACtE,SACE,IAAI,MAAM,cAAc,OAAO,YAAY,IAC3C,IAAI,MAAM,kBAAkB,OAAO,aAAa,IAChD,IAAI,MAAM,iBAAiB,OAAO,oBAAoB,OAAO,YAAY,IACzE,IAAI,MAAM,kBAAkB,OAAO,qBAAqB,OAAO,YAAY;AAE/E;AAUA,IAAM,UAAU,IAAI,0CAA+B;AAGnD,IAAM,kBAAN,MAA0C;AAAA,EACxC,YAA6B,OAAiB;AAAjB;AAAA,EAAkB;AAAA,EAE/C,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,MAAM,QAAQ,SAAS;AAC7B,QAAI,KAAK;AACP,UAAI,SAAS;AACb,eAAS,IAAI,OAAO,SAAS,KAAK;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AACF;AAEA,eAAe,mBACb,OACA,aACA,IACc;AACd,QAAM,UAAe,IAAI,MAAM,MAAM,MAAM;AAC3C,MAAI,OAAO;AACX,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,YAAY;AACtF,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,QAAQ;AACd,cAAQ,KAAK,IAAI,MAAM,GAAG,MAAM,KAAK,GAAG,KAAK;AAAA,IAC/C;AAAA,EACF,CAAC;AACD,QAAM,QAAQ,IAAI,OAAO;AACzB,SAAO;AACT;AAWA,eAAsB,QAAQ,SAA2C;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,IACb,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA,GAAG;AAAA,EACL,IAAI;AACJ,QAAM,UAAU,IAAI,gBAAgB,QAAQ;AAC5C,QAAM,MAAM,SAAS,WACjB,aAAa,mCAAuB,qBACpC,aAAa,0CAA8B;AAE/C,QAAM,QAAQ,MAAM,mBAAmB,UAAU,aAAa,OAAO,SAAS,UAAU;AACtF,UAAM,MAAmB,EAAE,OAAO,WAAW,GAAG,OAAO,EAAE;AACzD,UAAM,OAAO,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AACjD,UAAM,UAAU,YAAY,IAAI;AAChC,WAAO,QAAQ,IAAI,KAAK,YAA+B;AACrD,UAAI;AACF,cAAM,SAAS,MAAM,IAA6B,QAAQ,OAAO;AAAA,UAC/D,GAAG;AAAA,UACH,UAAU;AAAA,QACZ,CAAC;AACD,cAAM,OAAO,aACR,OAA6C,OAC7C;AACL,cAAM,OAAO,aAAc,OAA2D,aAAa,CAAC;AACpG,cAAM,SAAS,aAAc,OAAuC,SAAS,CAAC;AAC9E,cAAM,SAAS,cAAc,QAAQ,UAAU,MAAM,IAAI;AACzD,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,OAAO,MAAM,CAAC,MAAM,EAAE,YAAY,OAAO;AAAA,UAChD;AAAA,UACA;AAAA,UACA,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF,SAAS,OAAO;AACd,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC5D,OAAO;AAAA,UACP,QAAQ,cAAc,QAAQ,UAAU,MAAS;AAAA,UACjD,QAAQ,CAAC;AAAA,UACT,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,SAAS,WAAW,KAAK;AAC/B,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,QAAQ,WAAW;AACzB,aAAW,QAAQ,OAAO;AACxB,UAAM,gBAAgB,KAAK,MAAM;AACjC,UAAM,oBAAoB,KAAK,MAAM;AACrC,UAAM,mBAAmB,KAAK,MAAM;AACpC,UAAM,oBAAoB,KAAK,MAAM;AAAA,EACvC;AACA,QAAM,YAAY,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAEpE,SAAO;AAAA,IACL,UAAU,QAAQ,OAAO;AAAA,IACzB;AAAA,IACA,QAAO,oBAAI,KAAK,GAAE,YAAY;AAAA,IAC9B;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,UAAU,MAAM;AAAA,MAChB,IAAI,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE;AAAA,MAC9B,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAAA,MACpC,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,MACzB;AAAA,MACA,OAAO,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC;AAAA,MAC5C,GAAI,SAAS,EAAE,MAAM,aAAa,OAAO,MAAM,EAAE,IAAI,CAAC;AAAA,MACtD,WAAW;AAAA,QACT,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,UAAU,UAAU,SAAS,CAAC,KAAK;AAAA,QACxC,OAAO,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AACF;AAKO,SAAS,WAAW,QAAoB,MAAoB;AACjE,qCAAU,+BAAQ,2BAAQ,IAAI,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,yCAAc,2BAAQ,IAAI,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;AACrE;AAEO,SAAS,WAAW,MAAsC;AAC/D,QAAM,WAAO,2BAAQ,IAAI;AACzB,MAAI,KAAC,4BAAW,IAAI,EAAG,QAAO;AAC9B,SAAO,KAAK,UAAM,8BAAa,MAAM,MAAM,CAAC;AAC9C;AAsBA,SAAS,MAAM,GAA8B,GAA6C;AACxF,SAAO,MAAM,QAAQ,MAAM,UAAa,MAAM,QAAQ,MAAM,SAAY,OAAO,IAAI;AACrF;AAEO,SAAS,YAAY,UAAsB,MAA4B;AAC5E,QAAM,aAAa,IAAI,IAAI,SAAS,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,QAAM,SAAuB,CAAC;AAC9B,aAAW,SAAS,KAAK,QAAQ;AAC/B,UAAM,SAAS,WAAW,IAAI,MAAM,IAAI;AACxC,UAAM,YAAY,MAAM,QAAQ,WAAW,MAAM,SAAS;AAC1D,UAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,MAAM;AACjD,QAAK,cAAc,QAAQ,cAAc,KAAO,WAAW,QAAQ,WAAW,GAAI;AAChF,aAAO,KAAK,EAAE,MAAM,MAAM,MAAM,WAAW,OAAO,CAAC;AAAA,IACrD;AAAA,EACF;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,IAAI,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,CAAC;AAE3G,SAAO;AAAA,IACL,OAAO,KAAK,OAAO,QAAQ,SAAS,OAAO;AAAA,IAC3C,IAAI,KAAK,OAAO,KAAK,SAAS,OAAO;AAAA,IACrC,WAAW,MAAM,SAAS,OAAO,WAAW,KAAK,OAAO,SAAS;AAAA,IACjE,QAAQ,MAAM,SAAS,OAAO,QAAQ,KAAK,OAAO,MAAM;AAAA,IACxD,cAAc,KAAK,OAAO,MAAM,eAAe,SAAS,OAAO,MAAM;AAAA,IACrE,kBAAkB,KAAK,OAAO,MAAM,mBAAmB,SAAS,OAAO,MAAM;AAAA,IAC7E,GAAI,KAAK,OAAO,SAAS,UAAa,SAAS,OAAO,SAAS,SAC3D,EAAE,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO,KAAK,IAChD,CAAC;AAAA,IACL,OAAO,KAAK,OAAO,UAAU,MAAM,SAAS,OAAO,UAAU;AAAA,IAC7D;AAAA,EACF;AACF;AAKA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,eAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAC3E;AAEA,SAAS,OAAO,OAAsB,SAAS,GAAG,SAAS,IAAY;AACrE,MAAI,UAAU,QAAQ,UAAU,EAAG,QAAO;AAC1C,QAAM,OAAO,SAAS,IAAI,MAAM,QAAQ,MAAM,IAAI,OAAO,KAAK;AAC9D,SAAO,KAAK,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,GAAG,MAAM;AAClD;AAEA,SAAS,UAAU,OAA8B;AAC/C,SAAO,UAAU,QAAQ,UAAU,IAAI,KAAK,KAAK,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAClG;AAGO,SAAS,aAAa,QAAoB,MAAyB;AACxE,QAAM,IAAI,OAAO;AACjB,QAAM,QAAkB,CAAC;AACzB,QAAM;AAAA,IACJ,SAAS,OAAO,QAAQ,KAAK,OAAO,IAAI,YAAO,EAAE,QAAQ,gBAAgB,EAAE,EAAE,OAAO,OAAO,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,SAAS,OAAO,MAAM,SAAS,IAAI,CAAC;AAAA,EAC9J;AACA,QAAM,KAAK,EAAE;AAEb,QAAM,QAAQ,KAAK,IAAI,GAAG,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC;AACpE,QAAM,SAAS,IAAI,KAAK,MAAM,UAAU,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnE,QAAM,KAAK,GAAG,QAAQ,OAAO,KAAK,CAAC,qCAAgC;AACnE,aAAW,SAAS,OAAO,QAAQ;AACjC,UAAM,IAAI,OAAO,IAAI,MAAM,IAAI;AAC/B,UAAM,SAAS,IACX,CAAC,EAAE,cAAc,QAAQ,EAAE,cAAc,IAAI,IAAI,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,WAAW,QAAQ,EAAE,WAAW,IAAI,IAAI,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,EAC/J,OAAO,OAAO,EACd,KAAK,GAAG,IACX;AACJ,UAAM;AAAA,MACJ,GAAG,MAAM,KAAK,OAAO,KAAK,CAAC,KAAK,IAAI,MAAM,SAAS,CAAC,IAAI,IAAI,MAAM,MAAM,CAAC,MAAM,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,MAAM;AAAA,IAC1L;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,sBAAsB,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,aAAa,IAAI,CAAC,YAAY,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,UAAU,IAAI,CAAC;AAAA,EACtJ;AACA,QAAM,IAAI,EAAE;AACZ,QAAM,QAAQ,EAAE,mBAAmB,EAAE,mBACjC,gBAAgB,EAAE,gBAAgB,eAAe,OAAO,CAAC,YAAY,EAAE,iBAAiB,eAAe,OAAO,CAAC,KAC/G;AACJ,QAAM;AAAA,IACJ,WAAW,EAAE,aAAa,eAAe,OAAO,CAAC,UAAU,OAAO,MAAM,gBAAgB,IAAI,CAAC,MAAM,EAAE,iBAAiB,eAAe,OAAO,CAAC,cAAc,OAAO,MAAM,oBAAoB,IAAI,CAAC,GAAG,KAAK,KAAK,EAAE,KAAK,cAClN,EAAE,SAAS,SAAY,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAC,GAAG,OAAO,MAAM,QAAQ,MAAM,CAAC,CAAC,KAAK;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,gBAAgB,EAAE,UAAU,GAAG,KAAK,OAAO,MAAM,SAAS,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,EAAE,UAAU,GAAG;AAAA,EAC5H;AAEA,QAAM,SAAS,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE;AAC/C,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,SAAS;AACpB,eAAW,QAAQ,OAAQ,OAAM,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,EAAE;AAAA,EACvE;AACA,QAAM,YAAY,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK;AAC7D,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,aAAa;AACxB,eAAW,QAAQ,WAAW;AAC5B,iBAAW,QAAQ,KAAK,OAAO,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,GAAG;AACnE,cAAM,SACJ,KAAK,YAAY,UACb,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,SAAS,KAAK,UAAU,KAAK,MAAM,CAAC,KAC7E,KAAK,YAAY,YACf,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,kBACzC,cAAc,KAAK,UAAU,KAAK,MAAM,CAAC;AACjD,cAAM,OAAO,KAAK,aAAa,KAAK,KAAK,UAAU,MAAM;AACzD,cAAM,KAAK,KAAK,KAAK,IAAI,WAAM,KAAK,IAAI,KAAK,MAAM,GAAG,IAAI,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":["import_node_fs","import_node_path","stable"]}
+{"version":3,"sources":["../src/index.ts","../src/replay.ts","../src/eval.ts"],"sourcesContent":["export {\n  RecordingProvider,\n  ReplayProvider,\n  ReplayMissError,\n  replayOrRecord,\n  recordingKey,\n  recordingPath,\n} from \"./replay.js\";\nexport type { Recording, ReplayOptions } from \"./replay.js\";\n\nexport {\n  runEval,\n  loadFixtures,\n  compareLeaves,\n  flattenLeaves,\n  leavesEqual,\n  fieldStats,\n  estimateCost,\n  saveReport,\n  loadReport,\n  diffReports,\n  formatReport,\n} from \"./eval.js\";\nexport type {\n  EvalFixture,\n  EvalOptions,\n  EvalReport,\n  EvalItem,\n  EvalTotals,\n  EvalDiff,\n  FieldDelta,\n  FieldStats,\n  LeafResult,\n  LeafOutcome,\n  TokenPrices,\n  Usage,\n} from \"./eval.js\";\n","import { createHash } from \"node:crypto\";\nimport { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Provider, ProviderRequest, ProviderResponse } from \"@sembl/core\";\n\n/** What a recording holds: enough of the request to recognise it, and the answer. */\nexport interface Recording {\n  /** Hash of the parts of the request that determine the answer. */\n  key: string;\n  schemaId: string;\n  request: {\n    systemPrompt: string;\n    userInput: string;\n    jsonSchema: Record<string, unknown>;\n  };\n  response: ProviderResponse;\n  recordedAt: string;\n}\n\n/** JSON with sorted keys, so the same request always hashes the same. */\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_key, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\n/**\n * The key a request records under. Only the parts that reach the model\n * count: the system prompt, the user input and the JSON Schema. The runtime\n * schema and bundle are already folded into those.\n */\nexport function recordingKey(request: ProviderRequest): string {\n  const material = stable({\n    systemPrompt: request.systemPrompt,\n    userInput: request.userInput,\n    jsonSchema: request.jsonSchema,\n  });\n  return createHash(\"sha256\").update(material).digest(\"hex\").slice(0, 24);\n}\n\nfunction slug(id: string): string {\n  return id.replace(/[^A-Za-z0-9_-]+/g, \"_\").slice(0, 40) || \"schema\";\n}\n\n/** Where a recording for a request lives inside a directory. */\nexport function recordingPath(dir: string, request: ProviderRequest): string {\n  return join(dir, `${slug(request.schema.id)}.${recordingKey(request)}.json`);\n}\n\n/** Thrown by {@link ReplayProvider} when no recording matches a request. */\nexport class ReplayMissError extends Error {\n  constructor(\n    public readonly key: string,\n    public readonly dir: string,\n    public readonly schemaId: string,\n  ) {\n    super(\n      `No recording for a \"${schemaId}\" request (key ${key}) in ${dir}. ` +\n        \"Run once with a RecordingProvider, or pass a fallback provider to record misses.\",\n    );\n    this.name = \"ReplayMissError\";\n  }\n}\n\n/**\n * Calls another provider and writes every request/response pair to a\n * directory, one JSON file each, named by schema and request hash. Run your\n * extraction code through it once with real credentials; the files it leaves\n * behind let a {@link ReplayProvider} answer the same requests offline.\n *\n * A request is identified by what reaches the model, so editing a field\n * description or the input produces a new recording rather than a stale hit.\n */\nexport class RecordingProvider implements Provider {\n  constructor(\n    private readonly inner: Provider,\n    private readonly dir: string,\n  ) {\n    mkdirSync(dir, { recursive: true });\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const recording: Recording = {\n      key: recordingKey(request),\n      schemaId: request.schema.id,\n      request: {\n        systemPrompt: request.systemPrompt,\n        userInput: request.userInput,\n        jsonSchema: request.jsonSchema,\n      },\n      response,\n      recordedAt: new Date().toISOString(),\n    };\n    writeFileSync(recordingPath(this.dir, request), JSON.stringify(recording, null, 2) + \"\\n\");\n    return response;\n  }\n}\n\n/** Options for {@link ReplayProvider}. */\nexport interface ReplayOptions {\n  /**\n   * Where to send a request no recording matches. The answer is recorded,\n   * so the next run replays it. Without one, a miss throws\n   * {@link ReplayMissError} — the right behaviour in CI, where a miss means\n   * a fixture changed and nobody re-recorded.\n   */\n  fallback?: Provider;\n}\n\n/**\n * Answers requests from a directory of recordings, never touching the\n * network. Deterministic, free, and fast: the provider to put under tests of\n * your own extraction code.\n */\nexport class ReplayProvider implements Provider {\n  private readonly recorder: RecordingProvider | undefined;\n\n  constructor(\n    private readonly dir: string,\n    options: ReplayOptions = {},\n  ) {\n    this.recorder = options.fallback ? new RecordingProvider(options.fallback, dir) : undefined;\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const path = recordingPath(this.dir, request);\n    if (existsSync(path)) {\n      const recording = JSON.parse(readFileSync(path, \"utf8\")) as Recording;\n      return recording.response;\n    }\n    if (this.recorder) {\n      return this.recorder.complete(request);\n    }\n    throw new ReplayMissError(recordingKey(request), this.dir, request.schema.id);\n  }\n\n  /** How many recordings the directory holds. */\n  size(): number {\n    if (!existsSync(this.dir)) return 0;\n    return readdirSync(this.dir).filter((f) => f.endsWith(\".json\")).length;\n  }\n}\n\n/**\n * The usual arrangement: replay from `dir`, and record misses through\n * `live` when it is given — say, only when an API key is present — so the\n * same test file works locally with credentials and in CI without them.\n */\nexport function replayOrRecord(dir: string, live?: Provider): Provider {\n  return new ReplayProvider(dir, live ? { fallback: live } : {});\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\";\nimport { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from \"node:fs\";\nimport { basename, dirname, join, resolve } from \"node:path\";\nimport {\n  coerce,\n  partialCoerce,\n  coerceWithProvenance,\n  partialCoerceWithProvenance,\n} from \"@sembl/core\";\nimport type {\n  CoerceInput,\n  CoerceOptions,\n  FieldProvenance,\n  Provider,\n  ProviderRequest,\n  ProviderResponse,\n  ProviderUsage,\n  ResolvedIssue,\n  Source,\n} from \"@sembl/core\";\n\n/** One case: an input and what a correct extraction of it looks like. */\nexport interface EvalFixture {\n  /** Shown in the report. Defaults to the file it was loaded from. */\n  name?: string;\n  input: CoerceInput;\n  /** The expected extraction. Absent and `null` both mean \"not present\". */\n  expected: Record<string, unknown>;\n}\n\n/** Per-million-token prices, for a cost line in the report. */\nexport interface TokenPrices {\n  inputPerMTok: number;\n  outputPerMTok: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache reads. */\n  cacheReadPerMTok?: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache writes. */\n  cacheWritePerMTok?: number;\n}\n\n/** Options for {@link runEval}. Everything in `CoerceOptions` applies per fixture. */\nexport interface EvalOptions extends CoerceOptions {\n  fixtures: readonly EvalFixture[];\n  /** Which coercion to run. Default `\"coerce\"`. */\n  mode?: \"coerce\" | \"partialCoerce\";\n  /** Ask for provenance, so the report can show confidence per field. */\n  provenance?: boolean;\n  /** How many fixtures run at once. Default 1, for stable latency numbers. */\n  concurrency?: number;\n  /** Prices for the cost line. Without them the report has no cost. */\n  prices?: TokenPrices;\n}\n\n/** How one leaf compared. */\nexport type LeafOutcome = \"match\" | \"wrong\" | \"missing\" | \"extra\";\n\nexport interface LeafResult {\n  path: string;\n  outcome: LeafOutcome;\n  expected?: unknown;\n  actual?: unknown;\n  confidence?: FieldProvenance[\"confidence\"];\n}\n\nexport interface EvalItem {\n  name: string;\n  ok: boolean;\n  /** Message of the error the coercion threw, when it did. */\n  error?: string;\n  /** Whether every expected leaf matched and nothing extra came back. */\n  exact: boolean;\n  leaves: LeafResult[];\n  issues: ResolvedIssue[];\n  latencyMs: number;\n  calls: number;\n  usage: Usage;\n}\n\nexport interface Usage {\n  promptTokens: number;\n  completionTokens: number;\n  cacheReadTokens: number;\n  cacheWriteTokens: number;\n}\n\nexport interface FieldStats {\n  path: string;\n  tp: number;\n  fp: number;\n  fn: number;\n  /** `tp / (tp + fp)`; null when nothing was returned for the field. */\n  precision: number | null;\n  /** `tp / (tp + fn)`; null when nothing was expected for the field. */\n  recall: number | null;\n}\n\nexport interface EvalTotals {\n  fixtures: number;\n  ok: number;\n  exact: number;\n  precision: number | null;\n  recall: number | null;\n  usage: Usage;\n  calls: number;\n  cost?: number;\n  latencyMs: { p50: number; p95: number; max: number; total: number };\n}\n\nexport interface EvalReport {\n  schemaId: string;\n  mode: \"coerce\" | \"partialCoerce\";\n  ranAt: string;\n  items: EvalItem[];\n  fields: FieldStats[];\n  totals: EvalTotals;\n}\n\n// ---------------------------------------------------------------------------\n// Fixtures\n\ninterface FileInput {\n  file: string;\n  label?: string;\n}\n\nfunction isFileInput(value: unknown): value is FileInput {\n  return typeof value === \"object\" && value !== null && typeof (value as FileInput).file === \"string\";\n}\n\n/** Resolve `{ file, label }` inputs against the fixture's own directory. */\nfunction resolveInput(input: unknown, baseDir: string): CoerceInput {\n  const one = (value: unknown): Source | string => {\n    if (typeof value === \"string\") return value;\n    if (isFileInput(value)) {\n      const text = readFileSync(resolve(baseDir, value.file), \"utf8\");\n      return value.label ? { label: value.label, text } : { text };\n    }\n    return value as Source;\n  };\n  if (Array.isArray(input)) {\n    return input.map((v) => {\n      const r = one(v);\n      return typeof r === \"string\" ? { text: r } : r;\n    });\n  }\n  return one(input);\n}\n\n/**\n * Load fixtures from a directory: every `*.json` file is either one fixture\n * or an array of them. An input may be `{ \"file\": \"page.html\", \"label\": … }`\n * to pull text from a sibling file, which keeps large scraped pages out of\n * the JSON.\n */\nexport function loadFixtures(dir: string): EvalFixture[] {\n  const root = resolve(dir);\n  if (!existsSync(root)) {\n    throw new Error(`Fixture directory not found: ${root}`);\n  }\n  const fixtures: EvalFixture[] = [];\n  const files = readdirSync(root)\n    .filter((f) => f.endsWith(\".json\") && !f.startsWith(\".\"))\n    .sort();\n  for (const file of files) {\n    const path = join(root, file);\n    const parsed = JSON.parse(readFileSync(path, \"utf8\")) as unknown;\n    const list = Array.isArray(parsed) ? parsed : [parsed];\n    list.forEach((raw, index) => {\n      const fixture = raw as EvalFixture;\n      if (!fixture || typeof fixture !== \"object\" || !(\"input\" in fixture) || !(\"expected\" in fixture)) {\n        throw new Error(`${path}${list.length > 1 ? `[${index}]` : \"\"}: a fixture needs \"input\" and \"expected\"`);\n      }\n      const stem = basename(file, \".json\");\n      fixtures.push({\n        name: fixture.name ?? (list.length > 1 ? `${stem}[${index}]` : stem),\n        input: resolveInput(fixture.input, dirname(path)),\n        expected: fixture.expected,\n      });\n    });\n  }\n  return fixtures;\n}\n\n// ---------------------------------------------------------------------------\n// Scoring\n\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_k, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\nfunction isPrimitive(value: unknown): boolean {\n  return value === null || typeof value !== \"object\";\n}\n\n/**\n * Flatten a value to its leaves, keyed by dotted path. Objects recurse;\n * arrays are leaves. `null` and `undefined` are absence, not leaves.\n */\nexport function flattenLeaves(value: unknown, prefix = \"\"): Map<string, unknown> {\n  const leaves = new Map<string, unknown>();\n  if (value === null || value === undefined) return leaves;\n  if (typeof value !== \"object\" || Array.isArray(value)) {\n    leaves.set(prefix, value);\n    return leaves;\n  }\n  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {\n    const path = prefix ? `${prefix}.${key}` : key;\n    for (const [leafPath, leaf] of flattenLeaves(child, path)) {\n      leaves.set(leafPath, leaf);\n    }\n  }\n  return leaves;\n}\n\n/**\n * Whether two leaves agree. Arrays of primitives compare as multisets —\n * the order amenities come back in is not a fact about the listing —\n * and everything else compares structurally.\n */\nexport function leavesEqual(a: unknown, b: unknown): boolean {\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    if (a.every(isPrimitive) && b.every(isPrimitive)) {\n      const sortedA = [...a].map(String).sort();\n      const sortedB = [...b].map(String).sort();\n      return sortedA.every((v, i) => v === sortedB[i]);\n    }\n  }\n  return stable(a) === stable(b);\n}\n\n/** Compare one extraction against its expectation, leaf by leaf. */\nexport function compareLeaves(\n  expected: Record<string, unknown>,\n  actual: Record<string, unknown> | undefined,\n  provenance: Record<string, FieldProvenance> = {},\n): LeafResult[] {\n  const want = flattenLeaves(expected);\n  const got = flattenLeaves(actual ?? {});\n  const paths = [...new Set([...want.keys(), ...got.keys()])].sort();\n  return paths.map((path) => {\n    const top = path.split(\".\")[0];\n    const confidence = provenance[top]?.confidence;\n    const result: LeafResult = { path, outcome: \"match\" };\n    if (confidence) result.confidence = confidence;\n    if (want.has(path) && got.has(path)) {\n      result.expected = want.get(path);\n      result.actual = got.get(path);\n      result.outcome = leavesEqual(want.get(path), got.get(path)) ? \"match\" : \"wrong\";\n    } else if (want.has(path)) {\n      result.expected = want.get(path);\n      result.outcome = \"missing\";\n    } else {\n      result.actual = got.get(path);\n      result.outcome = \"extra\";\n    }\n    return result;\n  });\n}\n\nfunction ratio(num: number, den: number): number | null {\n  return den === 0 ? null : num / den;\n}\n\n/** Aggregate leaf outcomes into per-field precision and recall. */\nexport function fieldStats(items: readonly EvalItem[]): FieldStats[] {\n  const byPath = new Map<string, { tp: number; fp: number; fn: number }>();\n  for (const item of items) {\n    for (const leaf of item.leaves) {\n      const stats = byPath.get(leaf.path) ?? { tp: 0, fp: 0, fn: 0 };\n      switch (leaf.outcome) {\n        case \"match\":\n          stats.tp += 1;\n          break;\n        case \"wrong\":\n          stats.fp += 1;\n          stats.fn += 1;\n          break;\n        case \"missing\":\n          stats.fn += 1;\n          break;\n        case \"extra\":\n          stats.fp += 1;\n          break;\n      }\n      byPath.set(leaf.path, stats);\n    }\n  }\n  return [...byPath.entries()]\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([path, { tp, fp, fn }]) => ({\n      path,\n      tp,\n      fp,\n      fn,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n    }));\n}\n\nfunction percentile(sorted: readonly number[], p: number): number {\n  if (sorted.length === 0) return 0;\n  const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);\n  return sorted[Math.max(0, index)];\n}\n\nfunction emptyUsage(): Usage {\n  return { promptTokens: 0, completionTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };\n}\n\nfunction addUsage(into: Usage, usage: ProviderUsage | undefined): void {\n  if (!usage) return;\n  into.promptTokens += usage.promptTokens;\n  into.completionTokens += usage.completionTokens;\n  into.cacheReadTokens += usage.cacheReadTokens ?? 0;\n  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;\n}\n\n/** Dollars, from per-million-token prices. */\nexport function estimateCost(usage: Usage, prices: TokenPrices): number {\n  const per = (tokens: number, price: number) => (tokens / 1_000_000) * price;\n  return (\n    per(usage.promptTokens, prices.inputPerMTok) +\n    per(usage.completionTokens, prices.outputPerMTok) +\n    per(usage.cacheReadTokens, prices.cacheReadPerMTok ?? prices.inputPerMTok) +\n    per(usage.cacheWriteTokens, prices.cacheWritePerMTok ?? prices.inputPerMTok)\n  );\n}\n\n// ---------------------------------------------------------------------------\n// Running\n\ninterface ItemContext {\n  usage: Usage;\n  calls: number;\n}\n\nconst context = new AsyncLocalStorage<ItemContext>();\n\n/** Credits each call's usage to whichever fixture is running it. */\nclass MeteredProvider implements Provider {\n  constructor(private readonly inner: Provider) {}\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const ctx = context.getStore();\n    if (ctx) {\n      ctx.calls += 1;\n      addUsage(ctx.usage, response.usage);\n    }\n    return response;\n  }\n}\n\nasync function mapWithConcurrency<T, R>(\n  items: readonly T[],\n  concurrency: number,\n  fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results: R[] = new Array(items.length);\n  let next = 0;\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {\n    while (next < items.length) {\n      const index = next++;\n      results[index] = await fn(items[index], index);\n    }\n  });\n  await Promise.all(workers);\n  return results;\n}\n\n/**\n * Run every fixture through a coercion and score the results.\n *\n * Per-field precision and recall make a description change measurable: a\n * field whose recall dropped after a rewording is a field the rewording\n * hurt. Token usage and latency come along so the cost of a change is\n * visible next to its effect. Pair it with a `ReplayProvider` to run in CI\n * without spend.\n */\nexport async function runEval(options: EvalOptions): Promise<EvalReport> {\n  const {\n    fixtures,\n    mode = \"coerce\",\n    provenance = false,\n    concurrency = 1,\n    prices,\n    provider,\n    ...coerceOptions\n  } = options;\n  const metered = new MeteredProvider(provider);\n  const run = mode === \"coerce\"\n    ? provenance ? coerceWithProvenance : coerce\n    : provenance ? partialCoerceWithProvenance : partialCoerce;\n\n  const items = await mapWithConcurrency(fixtures, concurrency, async (fixture, index) => {\n    const ctx: ItemContext = { usage: emptyUsage(), calls: 0 };\n    const name = fixture.name ?? `fixture ${index + 1}`;\n    const started = performance.now();\n    return context.run(ctx, async (): Promise<EvalItem> => {\n      try {\n        const result = await run<Record<string, unknown>>(fixture.input, {\n          ...coerceOptions,\n          provider: metered,\n        });\n        const data = provenance\n          ? (result as { data: Record<string, unknown> }).data\n          : (result as Record<string, unknown>);\n        const prov = provenance ? (result as { provenance: Record<string, FieldProvenance> }).provenance : {};\n        const issues = provenance ? (result as { issues: ResolvedIssue[] }).issues : [];\n        const leaves = compareLeaves(fixture.expected, data, prov);\n        return {\n          name,\n          ok: true,\n          exact: leaves.every((l) => l.outcome === \"match\"),\n          leaves,\n          issues,\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      } catch (error) {\n        return {\n          name,\n          ok: false,\n          error: error instanceof Error ? error.message : String(error),\n          exact: false,\n          leaves: compareLeaves(fixture.expected, undefined),\n          issues: [],\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      }\n    });\n  });\n\n  const fields = fieldStats(items);\n  const tp = fields.reduce((s, f) => s + f.tp, 0);\n  const fp = fields.reduce((s, f) => s + f.fp, 0);\n  const fn = fields.reduce((s, f) => s + f.fn, 0);\n  const usage = emptyUsage();\n  for (const item of items) {\n    usage.promptTokens += item.usage.promptTokens;\n    usage.completionTokens += item.usage.completionTokens;\n    usage.cacheReadTokens += item.usage.cacheReadTokens;\n    usage.cacheWriteTokens += item.usage.cacheWriteTokens;\n  }\n  const latencies = items.map((i) => i.latencyMs).sort((a, b) => a - b);\n\n  return {\n    schemaId: options.schema.id,\n    mode,\n    ranAt: new Date().toISOString(),\n    items,\n    fields,\n    totals: {\n      fixtures: items.length,\n      ok: items.filter((i) => i.ok).length,\n      exact: items.filter((i) => i.exact).length,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n      usage,\n      calls: items.reduce((s, i) => s + i.calls, 0),\n      ...(prices ? { cost: estimateCost(usage, prices) } : {}),\n      latencyMs: {\n        p50: percentile(latencies, 50),\n        p95: percentile(latencies, 95),\n        max: latencies[latencies.length - 1] ?? 0,\n        total: latencies.reduce((s, l) => s + l, 0),\n      },\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Persistence and diffs\n\nexport function saveReport(report: EvalReport, file: string): void {\n  mkdirSync(dirname(resolve(file)), { recursive: true });\n  writeFileSync(resolve(file), JSON.stringify(report, null, 2) + \"\\n\");\n}\n\nexport function loadReport(file: string): EvalReport | undefined {\n  const path = resolve(file);\n  if (!existsSync(path)) return undefined;\n  return JSON.parse(readFileSync(path, \"utf8\")) as EvalReport;\n}\n\nexport interface FieldDelta {\n  path: string;\n  precision: number | null;\n  recall: number | null;\n}\n\n/** What changed between two runs. Deltas are `next − previous`. */\nexport interface EvalDiff {\n  exact: number;\n  ok: number;\n  precision: number | null;\n  recall: number | null;\n  promptTokens: number;\n  completionTokens: number;\n  cost?: number;\n  p50Ms: number;\n  /** Fields whose precision or recall moved, largest drop first. */\n  fields: FieldDelta[];\n}\n\nfunction delta(a: number | null | undefined, b: number | null | undefined): number | null {\n  return a === null || a === undefined || b === null || b === undefined ? null : b - a;\n}\n\nexport function diffReports(previous: EvalReport, next: EvalReport): EvalDiff {\n  const prevFields = new Map(previous.fields.map((f) => [f.path, f]));\n  const fields: FieldDelta[] = [];\n  for (const field of next.fields) {\n    const before = prevFields.get(field.path);\n    const precision = delta(before?.precision, field.precision);\n    const recall = delta(before?.recall, field.recall);\n    if ((precision !== null && precision !== 0) || (recall !== null && recall !== 0)) {\n      fields.push({ path: field.path, precision, recall });\n    }\n  }\n  fields.sort((a, b) => Math.min(a.precision ?? 0, a.recall ?? 0) - Math.min(b.precision ?? 0, b.recall ?? 0));\n\n  return {\n    exact: next.totals.exact - previous.totals.exact,\n    ok: next.totals.ok - previous.totals.ok,\n    precision: delta(previous.totals.precision, next.totals.precision),\n    recall: delta(previous.totals.recall, next.totals.recall),\n    promptTokens: next.totals.usage.promptTokens - previous.totals.usage.promptTokens,\n    completionTokens: next.totals.usage.completionTokens - previous.totals.usage.completionTokens,\n    ...(next.totals.cost !== undefined && previous.totals.cost !== undefined\n      ? { cost: next.totals.cost - previous.totals.cost }\n      : {}),\n    p50Ms: next.totals.latencyMs.p50 - previous.totals.latencyMs.p50,\n    fields,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Formatting\n\nfunction pct(value: number | null): string {\n  return value === null ? \"  –  \" : `${(value * 100).toFixed(0).padStart(3)}%`;\n}\n\nfunction signed(value: number | null, digits = 0, suffix = \"\"): string {\n  if (value === null || value === 0) return \"\";\n  const text = digits > 0 ? value.toFixed(digits) : String(value);\n  return ` (${value > 0 ? \"+\" : \"\"}${text}${suffix})`;\n}\n\nfunction signedPct(value: number | null): string {\n  return value === null || value === 0 ? \"\" : ` (${value > 0 ? \"+\" : \"\"}${(value * 100).toFixed(0)}pt)`;\n}\n\n/** A plain-text report for a terminal, with deltas when a previous run is given. */\nexport function formatReport(report: EvalReport, diff?: EvalDiff): string {\n  const t = report.totals;\n  const lines: string[] = [];\n  lines.push(\n    `Eval: ${report.schemaId} (${report.mode}) — ${t.fixtures} fixture(s), ${t.ok} ran${signed(diff?.ok ?? null)}, ${t.exact} exact${signed(diff?.exact ?? null)}`,\n  );\n  lines.push(\"\");\n\n  const width = Math.max(5, ...report.fields.map((f) => f.path.length));\n  const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));\n  lines.push(`${\"Field\".padEnd(width)}  Prec  Recall   tp  fp  fn  Δ`);\n  const col = (value: number | null, w: number) => pct(value).trim().padStart(w);\n  for (const field of report.fields) {\n    const d = deltas.get(field.path);\n    const change = d\n      ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : \"\", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : \"\"]\n          .filter(Boolean)\n          .join(\" \")\n      : \"\";\n    lines.push(\n      `${field.path.padEnd(width)}  ${col(field.precision, 4)}  ${col(field.recall, 6)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`,\n    );\n  }\n  lines.push(\"\");\n  lines.push(\n    `Overall: precision ${pct(t.precision).trim()}${signedPct(diff?.precision ?? null)}, recall ${pct(t.recall).trim()}${signedPct(diff?.recall ?? null)}`,\n  );\n  const u = t.usage;\n  const cache = u.cacheReadTokens || u.cacheWriteTokens\n    ? `, cache read ${u.cacheReadTokens.toLocaleString(\"en-US\")} / write ${u.cacheWriteTokens.toLocaleString(\"en-US\")}`\n    : \"\";\n  lines.push(\n    `Tokens: ${u.promptTokens.toLocaleString(\"en-US\")} prompt${signed(diff?.promptTokens ?? null)} / ${u.completionTokens.toLocaleString(\"en-US\")} completion${signed(diff?.completionTokens ?? null)}${cache}; ${t.calls} call(s)` +\n      (t.cost !== undefined ? `; cost $${t.cost.toFixed(4)}${signed(diff?.cost ?? null, 4)}` : \"\"),\n  );\n  lines.push(\n    `Latency: p50 ${t.latencyMs.p50}ms${signed(diff?.p50Ms ?? null, 0, \"ms\")}, p95 ${t.latencyMs.p95}ms, max ${t.latencyMs.max}ms`,\n  );\n\n  const failed = report.items.filter((i) => !i.ok);\n  if (failed.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Failed:\");\n    for (const item of failed) lines.push(`  ${item.name}: ${item.error}`);\n  }\n  const imperfect = report.items.filter((i) => i.ok && !i.exact);\n  if (imperfect.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Mismatches:\");\n    for (const item of imperfect) {\n      for (const leaf of item.leaves.filter((l) => l.outcome !== \"match\")) {\n        const detail =\n          leaf.outcome === \"wrong\"\n            ? `expected ${JSON.stringify(leaf.expected)}, got ${JSON.stringify(leaf.actual)}`\n            : leaf.outcome === \"missing\"\n              ? `expected ${JSON.stringify(leaf.expected)}, got nothing`\n              : `unexpected ${JSON.stringify(leaf.actual)}`;\n        const conf = leaf.confidence ? ` [${leaf.confidence}]` : \"\";\n        lines.push(`  ${item.name} › ${leaf.path}: ${detail}${conf}`);\n      }\n    }\n  }\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,yBAA2B;AAC3B,qBAAgF;AAChF,uBAAqB;AAkBrB,SAAS,OAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,MAAM,MAClC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAOO,SAAS,aAAa,SAAkC;AAC7D,QAAM,WAAW,OAAO;AAAA,IACtB,cAAc,QAAQ;AAAA,IACtB,WAAW,QAAQ;AAAA,IACnB,YAAY,QAAQ;AAAA,EACtB,CAAC;AACD,aAAO,+BAAW,QAAQ,EAAE,OAAO,QAAQ,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AACxE;AAEA,SAAS,KAAK,IAAoB;AAChC,SAAO,GAAG,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK;AAC7D;AAGO,SAAS,cAAc,KAAa,SAAkC;AAC3E,aAAO,uBAAK,KAAK,GAAG,KAAK,QAAQ,OAAO,EAAE,CAAC,IAAI,aAAa,OAAO,CAAC,OAAO;AAC7E;AAGO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EACzC,YACkB,KACA,KACA,UAChB;AACA;AAAA,MACE,uBAAuB,QAAQ,kBAAkB,GAAG,QAAQ,GAAG;AAAA,IAEjE;AAPgB;AACA;AACA;AAMhB,SAAK,OAAO;AAAA,EACd;AACF;AAWO,IAAM,oBAAN,MAA4C;AAAA,EACjD,YACmB,OACA,KACjB;AAFiB;AACA;AAEjB,kCAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,YAAuB;AAAA,MAC3B,KAAK,aAAa,OAAO;AAAA,MACzB,UAAU,QAAQ,OAAO;AAAA,MACzB,SAAS;AAAA,QACP,cAAc,QAAQ;AAAA,QACtB,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,MACA,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,sCAAc,cAAc,KAAK,KAAK,OAAO,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,IAAI,IAAI;AACzF,WAAO;AAAA,EACT;AACF;AAkBO,IAAM,iBAAN,MAAyC;AAAA,EAG9C,YACmB,KACjB,UAAyB,CAAC,GAC1B;AAFiB;AAGjB,SAAK,WAAW,QAAQ,WAAW,IAAI,kBAAkB,QAAQ,UAAU,GAAG,IAAI;AAAA,EACpF;AAAA,EAPiB;AAAA,EASjB,MAAM,SAAS,SAAqD;AAClE,UAAM,OAAO,cAAc,KAAK,KAAK,OAAO;AAC5C,YAAI,2BAAW,IAAI,GAAG;AACpB,YAAM,YAAY,KAAK,UAAM,6BAAa,MAAM,MAAM,CAAC;AACvD,aAAO,UAAU;AAAA,IACnB;AACA,QAAI,KAAK,UAAU;AACjB,aAAO,KAAK,SAAS,SAAS,OAAO;AAAA,IACvC;AACA,UAAM,IAAI,gBAAgB,aAAa,OAAO,GAAG,KAAK,KAAK,QAAQ,OAAO,EAAE;AAAA,EAC9E;AAAA;AAAA,EAGA,OAAe;AACb,QAAI,KAAC,2BAAW,KAAK,GAAG,EAAG,QAAO;AAClC,eAAO,4BAAY,KAAK,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,EAAE;AAAA,EAClE;AACF;AAOO,SAAS,eAAe,KAAa,MAA2B;AACrE,SAAO,IAAI,eAAe,KAAK,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC,CAAC;AAC/D;;;ACzJA,8BAAkC;AAClC,IAAAA,kBAAgF;AAChF,IAAAC,oBAAiD;AACjD,kBAKO;AAqHP,SAAS,YAAY,OAAoC;AACvD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAoB,SAAS;AAC7F;AAGA,SAAS,aAAa,OAAgB,SAA8B;AAClE,QAAM,MAAM,CAAC,UAAoC;AAC/C,QAAI,OAAO,UAAU,SAAU,QAAO;AACtC,QAAI,YAAY,KAAK,GAAG;AACtB,YAAM,WAAO,kCAAa,2BAAQ,SAAS,MAAM,IAAI,GAAG,MAAM;AAC9D,aAAO,MAAM,QAAQ,EAAE,OAAO,MAAM,OAAO,KAAK,IAAI,EAAE,KAAK;AAAA,IAC7D;AACA,WAAO;AAAA,EACT;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM,IAAI,CAAC,MAAM;AACtB,YAAM,IAAI,IAAI,CAAC;AACf,aAAO,OAAO,MAAM,WAAW,EAAE,MAAM,EAAE,IAAI;AAAA,IAC/C,CAAC;AAAA,EACH;AACA,SAAO,IAAI,KAAK;AAClB;AAQO,SAAS,aAAa,KAA4B;AACvD,QAAM,WAAO,2BAAQ,GAAG;AACxB,MAAI,KAAC,4BAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,gCAAgC,IAAI,EAAE;AAAA,EACxD;AACA,QAAM,WAA0B,CAAC;AACjC,QAAM,YAAQ,6BAAY,IAAI,EAC3B,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC,EACvD,KAAK;AACR,aAAW,QAAQ,OAAO;AACxB,UAAM,WAAO,wBAAK,MAAM,IAAI;AAC5B,UAAM,SAAS,KAAK,UAAM,8BAAa,MAAM,MAAM,CAAC;AACpD,UAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AACrD,SAAK,QAAQ,CAAC,KAAK,UAAU;AAC3B,YAAM,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,YAAY,YAAY,EAAE,WAAW,YAAY,EAAE,cAAc,UAAU;AAChG,cAAM,IAAI,MAAM,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,EAAE,0CAA0C;AAAA,MACzG;AACA,YAAM,WAAO,4BAAS,MAAM,OAAO;AACnC,eAAS,KAAK;AAAA,QACZ,MAAM,QAAQ,SAAS,KAAK,SAAS,IAAI,GAAG,IAAI,IAAI,KAAK,MAAM;AAAA,QAC/D,OAAO,aAAa,QAAQ,WAAO,2BAAQ,IAAI,CAAC;AAAA,QAChD,UAAU,QAAQ;AAAA,MACpB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAKA,SAASC,QAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,IAAI,MAChC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAEA,SAAS,YAAY,OAAyB;AAC5C,SAAO,UAAU,QAAQ,OAAO,UAAU;AAC5C;AAMO,SAAS,cAAc,OAAgB,SAAS,IAA0B;AAC/E,QAAM,SAAS,oBAAI,IAAqB;AACxC,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACrD,WAAO,IAAI,QAAQ,KAAK;AACxB,WAAO;AAAA,EACT;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAgC,GAAG;AAC3E,UAAM,OAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,eAAW,CAAC,UAAU,IAAI,KAAK,cAAc,OAAO,IAAI,GAAG;AACzD,aAAO,IAAI,UAAU,IAAI;AAAA,IAC3B;AAAA,EACF;AACA,SAAO;AACT;AAOO,SAAS,YAAY,GAAY,GAAqB;AAC3D,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,QAAI,EAAE,MAAM,WAAW,KAAK,EAAE,MAAM,WAAW,GAAG;AAChD,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,aAAO,QAAQ,MAAM,CAAC,GAAG,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,IACjD;AAAA,EACF;AACA,SAAOA,QAAO,CAAC,MAAMA,QAAO,CAAC;AAC/B;AAGO,SAAS,cACd,UACA,QACA,aAA8C,CAAC,GACjC;AACd,QAAM,OAAO,cAAc,QAAQ;AACnC,QAAM,MAAM,cAAc,UAAU,CAAC,CAAC;AACtC,QAAM,QAAQ,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,KAAK,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACjE,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,MAAM,KAAK,MAAM,GAAG,EAAE,CAAC;AAC7B,UAAM,aAAa,WAAW,GAAG,GAAG;AACpC,UAAM,SAAqB,EAAE,MAAM,SAAS,QAAQ;AACpD,QAAI,WAAY,QAAO,aAAa;AACpC,QAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG;AACnC,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU,YAAY,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,UAAU;AAAA,IAC1E,WAAW,KAAK,IAAI,IAAI,GAAG;AACzB,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,UAAU;AAAA,IACnB,OAAO;AACL,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU;AAAA,IACnB;AACA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,MAAM,KAAa,KAA4B;AACtD,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAGO,SAAS,WAAW,OAA0C;AACnE,QAAM,SAAS,oBAAI,IAAoD;AACvE,aAAW,QAAQ,OAAO;AACxB,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,QAAQ,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE;AAC7D,cAAQ,KAAK,SAAS;AAAA,QACpB,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,MACJ;AACA,aAAO,IAAI,KAAK,MAAM,KAAK;AAAA,IAC7B;AAAA,EACF;AACA,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EACxB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO;AAAA,IAChC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,IAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,EAC3B,EAAE;AACN;AAEA,SAAS,WAAW,QAA2B,GAAmB;AAChE,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,QAAQ,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,IAAI,CAAC;AAClF,SAAO,OAAO,KAAK,IAAI,GAAG,KAAK,CAAC;AAClC;AAEA,SAAS,aAAoB;AAC3B,SAAO,EAAE,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,kBAAkB,EAAE;AACzF;AAEA,SAAS,SAAS,MAAa,OAAwC;AACrE,MAAI,CAAC,MAAO;AACZ,OAAK,gBAAgB,MAAM;AAC3B,OAAK,oBAAoB,MAAM;AAC/B,OAAK,mBAAmB,MAAM,mBAAmB;AACjD,OAAK,oBAAoB,MAAM,oBAAoB;AACrD;AAGO,SAAS,aAAa,OAAc,QAA6B;AACtE,QAAM,MAAM,CAAC,QAAgB,UAAmB,SAAS,MAAa;AACtE,SACE,IAAI,MAAM,cAAc,OAAO,YAAY,IAC3C,IAAI,MAAM,kBAAkB,OAAO,aAAa,IAChD,IAAI,MAAM,iBAAiB,OAAO,oBAAoB,OAAO,YAAY,IACzE,IAAI,MAAM,kBAAkB,OAAO,qBAAqB,OAAO,YAAY;AAE/E;AAUA,IAAM,UAAU,IAAI,0CAA+B;AAGnD,IAAM,kBAAN,MAA0C;AAAA,EACxC,YAA6B,OAAiB;AAAjB;AAAA,EAAkB;AAAA,EAE/C,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,MAAM,QAAQ,SAAS;AAC7B,QAAI,KAAK;AACP,UAAI,SAAS;AACb,eAAS,IAAI,OAAO,SAAS,KAAK;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AACF;AAEA,eAAe,mBACb,OACA,aACA,IACc;AACd,QAAM,UAAe,IAAI,MAAM,MAAM,MAAM;AAC3C,MAAI,OAAO;AACX,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,YAAY;AACtF,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,QAAQ;AACd,cAAQ,KAAK,IAAI,MAAM,GAAG,MAAM,KAAK,GAAG,KAAK;AAAA,IAC/C;AAAA,EACF,CAAC;AACD,QAAM,QAAQ,IAAI,OAAO;AACzB,SAAO;AACT;AAWA,eAAsB,QAAQ,SAA2C;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,IACb,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA,GAAG;AAAA,EACL,IAAI;AACJ,QAAM,UAAU,IAAI,gBAAgB,QAAQ;AAC5C,QAAM,MAAM,SAAS,WACjB,aAAa,mCAAuB,qBACpC,aAAa,0CAA8B;AAE/C,QAAM,QAAQ,MAAM,mBAAmB,UAAU,aAAa,OAAO,SAAS,UAAU;AACtF,UAAM,MAAmB,EAAE,OAAO,WAAW,GAAG,OAAO,EAAE;AACzD,UAAM,OAAO,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AACjD,UAAM,UAAU,YAAY,IAAI;AAChC,WAAO,QAAQ,IAAI,KAAK,YAA+B;AACrD,UAAI;AACF,cAAM,SAAS,MAAM,IAA6B,QAAQ,OAAO;AAAA,UAC/D,GAAG;AAAA,UACH,UAAU;AAAA,QACZ,CAAC;AACD,cAAM,OAAO,aACR,OAA6C,OAC7C;AACL,cAAM,OAAO,aAAc,OAA2D,aAAa,CAAC;AACpG,cAAM,SAAS,aAAc,OAAuC,SAAS,CAAC;AAC9E,cAAM,SAAS,cAAc,QAAQ,UAAU,MAAM,IAAI;AACzD,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,OAAO,MAAM,CAAC,MAAM,EAAE,YAAY,OAAO;AAAA,UAChD;AAAA,UACA;AAAA,UACA,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF,SAAS,OAAO;AACd,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC5D,OAAO;AAAA,UACP,QAAQ,cAAc,QAAQ,UAAU,MAAS;AAAA,UACjD,QAAQ,CAAC;AAAA,UACT,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,SAAS,WAAW,KAAK;AAC/B,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,QAAQ,WAAW;AACzB,aAAW,QAAQ,OAAO;AACxB,UAAM,gBAAgB,KAAK,MAAM;AACjC,UAAM,oBAAoB,KAAK,MAAM;AACrC,UAAM,mBAAmB,KAAK,MAAM;AACpC,UAAM,oBAAoB,KAAK,MAAM;AAAA,EACvC;AACA,QAAM,YAAY,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAEpE,SAAO;AAAA,IACL,UAAU,QAAQ,OAAO;AAAA,IACzB;AAAA,IACA,QAAO,oBAAI,KAAK,GAAE,YAAY;AAAA,IAC9B;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,UAAU,MAAM;AAAA,MAChB,IAAI,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE;AAAA,MAC9B,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAAA,MACpC,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,MACzB;AAAA,MACA,OAAO,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC;AAAA,MAC5C,GAAI,SAAS,EAAE,MAAM,aAAa,OAAO,MAAM,EAAE,IAAI,CAAC;AAAA,MACtD,WAAW;AAAA,QACT,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,UAAU,UAAU,SAAS,CAAC,KAAK;AAAA,QACxC,OAAO,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AACF;AAKO,SAAS,WAAW,QAAoB,MAAoB;AACjE,qCAAU,+BAAQ,2BAAQ,IAAI,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,yCAAc,2BAAQ,IAAI,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;AACrE;AAEO,SAAS,WAAW,MAAsC;AAC/D,QAAM,WAAO,2BAAQ,IAAI;AACzB,MAAI,KAAC,4BAAW,IAAI,EAAG,QAAO;AAC9B,SAAO,KAAK,UAAM,8BAAa,MAAM,MAAM,CAAC;AAC9C;AAsBA,SAAS,MAAM,GAA8B,GAA6C;AACxF,SAAO,MAAM,QAAQ,MAAM,UAAa,MAAM,QAAQ,MAAM,SAAY,OAAO,IAAI;AACrF;AAEO,SAAS,YAAY,UAAsB,MAA4B;AAC5E,QAAM,aAAa,IAAI,IAAI,SAAS,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,QAAM,SAAuB,CAAC;AAC9B,aAAW,SAAS,KAAK,QAAQ;AAC/B,UAAM,SAAS,WAAW,IAAI,MAAM,IAAI;AACxC,UAAM,YAAY,MAAM,QAAQ,WAAW,MAAM,SAAS;AAC1D,UAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,MAAM;AACjD,QAAK,cAAc,QAAQ,cAAc,KAAO,WAAW,QAAQ,WAAW,GAAI;AAChF,aAAO,KAAK,EAAE,MAAM,MAAM,MAAM,WAAW,OAAO,CAAC;AAAA,IACrD;AAAA,EACF;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,IAAI,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,CAAC;AAE3G,SAAO;AAAA,IACL,OAAO,KAAK,OAAO,QAAQ,SAAS,OAAO;AAAA,IAC3C,IAAI,KAAK,OAAO,KAAK,SAAS,OAAO;AAAA,IACrC,WAAW,MAAM,SAAS,OAAO,WAAW,KAAK,OAAO,SAAS;AAAA,IACjE,QAAQ,MAAM,SAAS,OAAO,QAAQ,KAAK,OAAO,MAAM;AAAA,IACxD,cAAc,KAAK,OAAO,MAAM,eAAe,SAAS,OAAO,MAAM;AAAA,IACrE,kBAAkB,KAAK,OAAO,MAAM,mBAAmB,SAAS,OAAO,MAAM;AAAA,IAC7E,GAAI,KAAK,OAAO,SAAS,UAAa,SAAS,OAAO,SAAS,SAC3D,EAAE,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO,KAAK,IAChD,CAAC;AAAA,IACL,OAAO,KAAK,OAAO,UAAU,MAAM,SAAS,OAAO,UAAU;AAAA,IAC7D;AAAA,EACF;AACF;AAKA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,eAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAC3E;AAEA,SAAS,OAAO,OAAsB,SAAS,GAAG,SAAS,IAAY;AACrE,MAAI,UAAU,QAAQ,UAAU,EAAG,QAAO;AAC1C,QAAM,OAAO,SAAS,IAAI,MAAM,QAAQ,MAAM,IAAI,OAAO,KAAK;AAC9D,SAAO,KAAK,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,GAAG,MAAM;AAClD;AAEA,SAAS,UAAU,OAA8B;AAC/C,SAAO,UAAU,QAAQ,UAAU,IAAI,KAAK,KAAK,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAClG;AAGO,SAAS,aAAa,QAAoB,MAAyB;AACxE,QAAM,IAAI,OAAO;AACjB,QAAM,QAAkB,CAAC;AACzB,QAAM;AAAA,IACJ,SAAS,OAAO,QAAQ,KAAK,OAAO,IAAI,YAAO,EAAE,QAAQ,gBAAgB,EAAE,EAAE,OAAO,OAAO,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,SAAS,OAAO,MAAM,SAAS,IAAI,CAAC;AAAA,EAC9J;AACA,QAAM,KAAK,EAAE;AAEb,QAAM,QAAQ,KAAK,IAAI,GAAG,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC;AACpE,QAAM,SAAS,IAAI,KAAK,MAAM,UAAU,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnE,QAAM,KAAK,GAAG,QAAQ,OAAO,KAAK,CAAC,qCAAgC;AACnE,QAAM,MAAM,CAAC,OAAsB,MAAc,IAAI,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC;AAC7E,aAAW,SAAS,OAAO,QAAQ;AACjC,UAAM,IAAI,OAAO,IAAI,MAAM,IAAI;AAC/B,UAAM,SAAS,IACX,CAAC,EAAE,cAAc,QAAQ,EAAE,cAAc,IAAI,IAAI,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,WAAW,QAAQ,EAAE,WAAW,IAAI,IAAI,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,EAC/J,OAAO,OAAO,EACd,KAAK,GAAG,IACX;AACJ,UAAM;AAAA,MACJ,GAAG,MAAM,KAAK,OAAO,KAAK,CAAC,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC,KAAK,IAAI,MAAM,QAAQ,CAAC,CAAC,MAAM,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,MAAM;AAAA,IACjM;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,sBAAsB,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,aAAa,IAAI,CAAC,YAAY,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,UAAU,IAAI,CAAC;AAAA,EACtJ;AACA,QAAM,IAAI,EAAE;AACZ,QAAM,QAAQ,EAAE,mBAAmB,EAAE,mBACjC,gBAAgB,EAAE,gBAAgB,eAAe,OAAO,CAAC,YAAY,EAAE,iBAAiB,eAAe,OAAO,CAAC,KAC/G;AACJ,QAAM;AAAA,IACJ,WAAW,EAAE,aAAa,eAAe,OAAO,CAAC,UAAU,OAAO,MAAM,gBAAgB,IAAI,CAAC,MAAM,EAAE,iBAAiB,eAAe,OAAO,CAAC,cAAc,OAAO,MAAM,oBAAoB,IAAI,CAAC,GAAG,KAAK,KAAK,EAAE,KAAK,cAClN,EAAE,SAAS,SAAY,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAC,GAAG,OAAO,MAAM,QAAQ,MAAM,CAAC,CAAC,KAAK;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,gBAAgB,EAAE,UAAU,GAAG,KAAK,OAAO,MAAM,SAAS,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,EAAE,UAAU,GAAG;AAAA,EAC5H;AAEA,QAAM,SAAS,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE;AAC/C,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,SAAS;AACpB,eAAW,QAAQ,OAAQ,OAAM,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,EAAE;AAAA,EACvE;AACA,QAAM,YAAY,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK;AAC7D,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,aAAa;AACxB,eAAW,QAAQ,WAAW;AAC5B,iBAAW,QAAQ,KAAK,OAAO,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,GAAG;AACnE,cAAM,SACJ,KAAK,YAAY,UACb,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,SAAS,KAAK,UAAU,KAAK,MAAM,CAAC,KAC7E,KAAK,YAAY,YACf,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,kBACzC,cAAc,KAAK,UAAU,KAAK,MAAM,CAAC;AACjD,cAAM,OAAO,KAAK,aAAa,KAAK,KAAK,UAAU,MAAM;AACzD,cAAM,KAAK,KAAK,KAAK,IAAI,WAAM,KAAK,IAAI,KAAK,MAAM,GAAG,IAAI,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":["import_node_fs","import_node_path","stable"]}

package/dist/index.js

@@ -422,11 +422,12 @@ function formatReport(report, diff) {
   const width = Math.max(5, ...report.fields.map((f) => f.path.length));
   const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));
   lines.push(`${"Field".padEnd(width)}  Prec  Recall   tp  fp  fn  \u0394`);
+  const col = (value, w) => pct(value).trim().padStart(w);
   for (const field of report.fields) {
     const d = deltas.get(field.path);
     const change = d ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : "", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : ""].filter(Boolean).join(" ") : "";
     lines.push(
-      `${field.path.padEnd(width)}  ${pct(field.precision)} ${pct(field.recall)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`
+      `${field.path.padEnd(width)}  ${col(field.precision, 4)}  ${col(field.recall, 6)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`
     );
   }
   lines.push("");

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/replay.ts","../src/eval.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Provider, ProviderRequest, ProviderResponse } from \"@sembl/core\";\n\n/** What a recording holds: enough of the request to recognise it, and the answer. */\nexport interface Recording {\n  /** Hash of the parts of the request that determine the answer. */\n  key: string;\n  schemaId: string;\n  request: {\n    systemPrompt: string;\n    userInput: string;\n    jsonSchema: Record<string, unknown>;\n  };\n  response: ProviderResponse;\n  recordedAt: string;\n}\n\n/** JSON with sorted keys, so the same request always hashes the same. */\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_key, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\n/**\n * The key a request records under. Only the parts that reach the model\n * count: the system prompt, the user input and the JSON Schema. The runtime\n * schema and bundle are already folded into those.\n */\nexport function recordingKey(request: ProviderRequest): string {\n  const material = stable({\n    systemPrompt: request.systemPrompt,\n    userInput: request.userInput,\n    jsonSchema: request.jsonSchema,\n  });\n  return createHash(\"sha256\").update(material).digest(\"hex\").slice(0, 24);\n}\n\nfunction slug(id: string): string {\n  return id.replace(/[^A-Za-z0-9_-]+/g, \"_\").slice(0, 40) || \"schema\";\n}\n\n/** Where a recording for a request lives inside a directory. */\nexport function recordingPath(dir: string, request: ProviderRequest): string {\n  return join(dir, `${slug(request.schema.id)}.${recordingKey(request)}.json`);\n}\n\n/** Thrown by {@link ReplayProvider} when no recording matches a request. */\nexport class ReplayMissError extends Error {\n  constructor(\n    public readonly key: string,\n    public readonly dir: string,\n    public readonly schemaId: string,\n  ) {\n    super(\n      `No recording for a \"${schemaId}\" request (key ${key}) in ${dir}. ` +\n        \"Run once with a RecordingProvider, or pass a fallback provider to record misses.\",\n    );\n    this.name = \"ReplayMissError\";\n  }\n}\n\n/**\n * Calls another provider and writes every request/response pair to a\n * directory, one JSON file each, named by schema and request hash. Run your\n * extraction code through it once with real credentials; the files it leaves\n * behind let a {@link ReplayProvider} answer the same requests offline.\n *\n * A request is identified by what reaches the model, so editing a field\n * description or the input produces a new recording rather than a stale hit.\n */\nexport class RecordingProvider implements Provider {\n  constructor(\n    private readonly inner: Provider,\n    private readonly dir: string,\n  ) {\n    mkdirSync(dir, { recursive: true });\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const recording: Recording = {\n      key: recordingKey(request),\n      schemaId: request.schema.id,\n      request: {\n        systemPrompt: request.systemPrompt,\n        userInput: request.userInput,\n        jsonSchema: request.jsonSchema,\n      },\n      response,\n      recordedAt: new Date().toISOString(),\n    };\n    writeFileSync(recordingPath(this.dir, request), JSON.stringify(recording, null, 2) + \"\\n\");\n    return response;\n  }\n}\n\n/** Options for {@link ReplayProvider}. */\nexport interface ReplayOptions {\n  /**\n   * Where to send a request no recording matches. The answer is recorded,\n   * so the next run replays it. Without one, a miss throws\n   * {@link ReplayMissError} — the right behaviour in CI, where a miss means\n   * a fixture changed and nobody re-recorded.\n   */\n  fallback?: Provider;\n}\n\n/**\n * Answers requests from a directory of recordings, never touching the\n * network. Deterministic, free, and fast: the provider to put under tests of\n * your own extraction code.\n */\nexport class ReplayProvider implements Provider {\n  private readonly recorder: RecordingProvider | undefined;\n\n  constructor(\n    private readonly dir: string,\n    options: ReplayOptions = {},\n  ) {\n    this.recorder = options.fallback ? new RecordingProvider(options.fallback, dir) : undefined;\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const path = recordingPath(this.dir, request);\n    if (existsSync(path)) {\n      const recording = JSON.parse(readFileSync(path, \"utf8\")) as Recording;\n      return recording.response;\n    }\n    if (this.recorder) {\n      return this.recorder.complete(request);\n    }\n    throw new ReplayMissError(recordingKey(request), this.dir, request.schema.id);\n  }\n\n  /** How many recordings the directory holds. */\n  size(): number {\n    if (!existsSync(this.dir)) return 0;\n    return readdirSync(this.dir).filter((f) => f.endsWith(\".json\")).length;\n  }\n}\n\n/**\n * The usual arrangement: replay from `dir`, and record misses through\n * `live` when it is given — say, only when an API key is present — so the\n * same test file works locally with credentials and in CI without them.\n */\nexport function replayOrRecord(dir: string, live?: Provider): Provider {\n  return new ReplayProvider(dir, live ? { fallback: live } : {});\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\";\nimport { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from \"node:fs\";\nimport { basename, dirname, join, resolve } from \"node:path\";\nimport {\n  coerce,\n  partialCoerce,\n  coerceWithProvenance,\n  partialCoerceWithProvenance,\n} from \"@sembl/core\";\nimport type {\n  CoerceInput,\n  CoerceOptions,\n  FieldProvenance,\n  Provider,\n  ProviderRequest,\n  ProviderResponse,\n  ProviderUsage,\n  ResolvedIssue,\n  Source,\n} from \"@sembl/core\";\n\n/** One case: an input and what a correct extraction of it looks like. */\nexport interface EvalFixture {\n  /** Shown in the report. Defaults to the file it was loaded from. */\n  name?: string;\n  input: CoerceInput;\n  /** The expected extraction. Absent and `null` both mean \"not present\". */\n  expected: Record<string, unknown>;\n}\n\n/** Per-million-token prices, for a cost line in the report. */\nexport interface TokenPrices {\n  inputPerMTok: number;\n  outputPerMTok: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache reads. */\n  cacheReadPerMTok?: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache writes. */\n  cacheWritePerMTok?: number;\n}\n\n/** Options for {@link runEval}. Everything in `CoerceOptions` applies per fixture. */\nexport interface EvalOptions extends CoerceOptions {\n  fixtures: readonly EvalFixture[];\n  /** Which coercion to run. Default `\"coerce\"`. */\n  mode?: \"coerce\" | \"partialCoerce\";\n  /** Ask for provenance, so the report can show confidence per field. */\n  provenance?: boolean;\n  /** How many fixtures run at once. Default 1, for stable latency numbers. */\n  concurrency?: number;\n  /** Prices for the cost line. Without them the report has no cost. */\n  prices?: TokenPrices;\n}\n\n/** How one leaf compared. */\nexport type LeafOutcome = \"match\" | \"wrong\" | \"missing\" | \"extra\";\n\nexport interface LeafResult {\n  path: string;\n  outcome: LeafOutcome;\n  expected?: unknown;\n  actual?: unknown;\n  confidence?: FieldProvenance[\"confidence\"];\n}\n\nexport interface EvalItem {\n  name: string;\n  ok: boolean;\n  /** Message of the error the coercion threw, when it did. */\n  error?: string;\n  /** Whether every expected leaf matched and nothing extra came back. */\n  exact: boolean;\n  leaves: LeafResult[];\n  issues: ResolvedIssue[];\n  latencyMs: number;\n  calls: number;\n  usage: Usage;\n}\n\nexport interface Usage {\n  promptTokens: number;\n  completionTokens: number;\n  cacheReadTokens: number;\n  cacheWriteTokens: number;\n}\n\nexport interface FieldStats {\n  path: string;\n  tp: number;\n  fp: number;\n  fn: number;\n  /** `tp / (tp + fp)`; null when nothing was returned for the field. */\n  precision: number | null;\n  /** `tp / (tp + fn)`; null when nothing was expected for the field. */\n  recall: number | null;\n}\n\nexport interface EvalTotals {\n  fixtures: number;\n  ok: number;\n  exact: number;\n  precision: number | null;\n  recall: number | null;\n  usage: Usage;\n  calls: number;\n  cost?: number;\n  latencyMs: { p50: number; p95: number; max: number; total: number };\n}\n\nexport interface EvalReport {\n  schemaId: string;\n  mode: \"coerce\" | \"partialCoerce\";\n  ranAt: string;\n  items: EvalItem[];\n  fields: FieldStats[];\n  totals: EvalTotals;\n}\n\n// ---------------------------------------------------------------------------\n// Fixtures\n\ninterface FileInput {\n  file: string;\n  label?: string;\n}\n\nfunction isFileInput(value: unknown): value is FileInput {\n  return typeof value === \"object\" && value !== null && typeof (value as FileInput).file === \"string\";\n}\n\n/** Resolve `{ file, label }` inputs against the fixture's own directory. */\nfunction resolveInput(input: unknown, baseDir: string): CoerceInput {\n  const one = (value: unknown): Source | string => {\n    if (typeof value === \"string\") return value;\n    if (isFileInput(value)) {\n      const text = readFileSync(resolve(baseDir, value.file), \"utf8\");\n      return value.label ? { label: value.label, text } : { text };\n    }\n    return value as Source;\n  };\n  if (Array.isArray(input)) {\n    return input.map((v) => {\n      const r = one(v);\n      return typeof r === \"string\" ? { text: r } : r;\n    });\n  }\n  return one(input);\n}\n\n/**\n * Load fixtures from a directory: every `*.json` file is either one fixture\n * or an array of them. An input may be `{ \"file\": \"page.html\", \"label\": … }`\n * to pull text from a sibling file, which keeps large scraped pages out of\n * the JSON.\n */\nexport function loadFixtures(dir: string): EvalFixture[] {\n  const root = resolve(dir);\n  if (!existsSync(root)) {\n    throw new Error(`Fixture directory not found: ${root}`);\n  }\n  const fixtures: EvalFixture[] = [];\n  const files = readdirSync(root)\n    .filter((f) => f.endsWith(\".json\") && !f.startsWith(\".\"))\n    .sort();\n  for (const file of files) {\n    const path = join(root, file);\n    const parsed = JSON.parse(readFileSync(path, \"utf8\")) as unknown;\n    const list = Array.isArray(parsed) ? parsed : [parsed];\n    list.forEach((raw, index) => {\n      const fixture = raw as EvalFixture;\n      if (!fixture || typeof fixture !== \"object\" || !(\"input\" in fixture) || !(\"expected\" in fixture)) {\n        throw new Error(`${path}${list.length > 1 ? `[${index}]` : \"\"}: a fixture needs \"input\" and \"expected\"`);\n      }\n      const stem = basename(file, \".json\");\n      fixtures.push({\n        name: fixture.name ?? (list.length > 1 ? `${stem}[${index}]` : stem),\n        input: resolveInput(fixture.input, dirname(path)),\n        expected: fixture.expected,\n      });\n    });\n  }\n  return fixtures;\n}\n\n// ---------------------------------------------------------------------------\n// Scoring\n\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_k, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\nfunction isPrimitive(value: unknown): boolean {\n  return value === null || typeof value !== \"object\";\n}\n\n/**\n * Flatten a value to its leaves, keyed by dotted path. Objects recurse;\n * arrays are leaves. `null` and `undefined` are absence, not leaves.\n */\nexport function flattenLeaves(value: unknown, prefix = \"\"): Map<string, unknown> {\n  const leaves = new Map<string, unknown>();\n  if (value === null || value === undefined) return leaves;\n  if (typeof value !== \"object\" || Array.isArray(value)) {\n    leaves.set(prefix, value);\n    return leaves;\n  }\n  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {\n    const path = prefix ? `${prefix}.${key}` : key;\n    for (const [leafPath, leaf] of flattenLeaves(child, path)) {\n      leaves.set(leafPath, leaf);\n    }\n  }\n  return leaves;\n}\n\n/**\n * Whether two leaves agree. Arrays of primitives compare as multisets —\n * the order amenities come back in is not a fact about the listing —\n * and everything else compares structurally.\n */\nexport function leavesEqual(a: unknown, b: unknown): boolean {\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    if (a.every(isPrimitive) && b.every(isPrimitive)) {\n      const sortedA = [...a].map(String).sort();\n      const sortedB = [...b].map(String).sort();\n      return sortedA.every((v, i) => v === sortedB[i]);\n    }\n  }\n  return stable(a) === stable(b);\n}\n\n/** Compare one extraction against its expectation, leaf by leaf. */\nexport function compareLeaves(\n  expected: Record<string, unknown>,\n  actual: Record<string, unknown> | undefined,\n  provenance: Record<string, FieldProvenance> = {},\n): LeafResult[] {\n  const want = flattenLeaves(expected);\n  const got = flattenLeaves(actual ?? {});\n  const paths = [...new Set([...want.keys(), ...got.keys()])].sort();\n  return paths.map((path) => {\n    const top = path.split(\".\")[0];\n    const confidence = provenance[top]?.confidence;\n    const result: LeafResult = { path, outcome: \"match\" };\n    if (confidence) result.confidence = confidence;\n    if (want.has(path) && got.has(path)) {\n      result.expected = want.get(path);\n      result.actual = got.get(path);\n      result.outcome = leavesEqual(want.get(path), got.get(path)) ? \"match\" : \"wrong\";\n    } else if (want.has(path)) {\n      result.expected = want.get(path);\n      result.outcome = \"missing\";\n    } else {\n      result.actual = got.get(path);\n      result.outcome = \"extra\";\n    }\n    return result;\n  });\n}\n\nfunction ratio(num: number, den: number): number | null {\n  return den === 0 ? null : num / den;\n}\n\n/** Aggregate leaf outcomes into per-field precision and recall. */\nexport function fieldStats(items: readonly EvalItem[]): FieldStats[] {\n  const byPath = new Map<string, { tp: number; fp: number; fn: number }>();\n  for (const item of items) {\n    for (const leaf of item.leaves) {\n      const stats = byPath.get(leaf.path) ?? { tp: 0, fp: 0, fn: 0 };\n      switch (leaf.outcome) {\n        case \"match\":\n          stats.tp += 1;\n          break;\n        case \"wrong\":\n          stats.fp += 1;\n          stats.fn += 1;\n          break;\n        case \"missing\":\n          stats.fn += 1;\n          break;\n        case \"extra\":\n          stats.fp += 1;\n          break;\n      }\n      byPath.set(leaf.path, stats);\n    }\n  }\n  return [...byPath.entries()]\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([path, { tp, fp, fn }]) => ({\n      path,\n      tp,\n      fp,\n      fn,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n    }));\n}\n\nfunction percentile(sorted: readonly number[], p: number): number {\n  if (sorted.length === 0) return 0;\n  const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);\n  return sorted[Math.max(0, index)];\n}\n\nfunction emptyUsage(): Usage {\n  return { promptTokens: 0, completionTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };\n}\n\nfunction addUsage(into: Usage, usage: ProviderUsage | undefined): void {\n  if (!usage) return;\n  into.promptTokens += usage.promptTokens;\n  into.completionTokens += usage.completionTokens;\n  into.cacheReadTokens += usage.cacheReadTokens ?? 0;\n  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;\n}\n\n/** Dollars, from per-million-token prices. */\nexport function estimateCost(usage: Usage, prices: TokenPrices): number {\n  const per = (tokens: number, price: number) => (tokens / 1_000_000) * price;\n  return (\n    per(usage.promptTokens, prices.inputPerMTok) +\n    per(usage.completionTokens, prices.outputPerMTok) +\n    per(usage.cacheReadTokens, prices.cacheReadPerMTok ?? prices.inputPerMTok) +\n    per(usage.cacheWriteTokens, prices.cacheWritePerMTok ?? prices.inputPerMTok)\n  );\n}\n\n// ---------------------------------------------------------------------------\n// Running\n\ninterface ItemContext {\n  usage: Usage;\n  calls: number;\n}\n\nconst context = new AsyncLocalStorage<ItemContext>();\n\n/** Credits each call's usage to whichever fixture is running it. */\nclass MeteredProvider implements Provider {\n  constructor(private readonly inner: Provider) {}\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const ctx = context.getStore();\n    if (ctx) {\n      ctx.calls += 1;\n      addUsage(ctx.usage, response.usage);\n    }\n    return response;\n  }\n}\n\nasync function mapWithConcurrency<T, R>(\n  items: readonly T[],\n  concurrency: number,\n  fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results: R[] = new Array(items.length);\n  let next = 0;\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {\n    while (next < items.length) {\n      const index = next++;\n      results[index] = await fn(items[index], index);\n    }\n  });\n  await Promise.all(workers);\n  return results;\n}\n\n/**\n * Run every fixture through a coercion and score the results.\n *\n * Per-field precision and recall make a description change measurable: a\n * field whose recall dropped after a rewording is a field the rewording\n * hurt. Token usage and latency come along so the cost of a change is\n * visible next to its effect. Pair it with a `ReplayProvider` to run in CI\n * without spend.\n */\nexport async function runEval(options: EvalOptions): Promise<EvalReport> {\n  const {\n    fixtures,\n    mode = \"coerce\",\n    provenance = false,\n    concurrency = 1,\n    prices,\n    provider,\n    ...coerceOptions\n  } = options;\n  const metered = new MeteredProvider(provider);\n  const run = mode === \"coerce\"\n    ? provenance ? coerceWithProvenance : coerce\n    : provenance ? partialCoerceWithProvenance : partialCoerce;\n\n  const items = await mapWithConcurrency(fixtures, concurrency, async (fixture, index) => {\n    const ctx: ItemContext = { usage: emptyUsage(), calls: 0 };\n    const name = fixture.name ?? `fixture ${index + 1}`;\n    const started = performance.now();\n    return context.run(ctx, async (): Promise<EvalItem> => {\n      try {\n        const result = await run<Record<string, unknown>>(fixture.input, {\n          ...coerceOptions,\n          provider: metered,\n        });\n        const data = provenance\n          ? (result as { data: Record<string, unknown> }).data\n          : (result as Record<string, unknown>);\n        const prov = provenance ? (result as { provenance: Record<string, FieldProvenance> }).provenance : {};\n        const issues = provenance ? (result as { issues: ResolvedIssue[] }).issues : [];\n        const leaves = compareLeaves(fixture.expected, data, prov);\n        return {\n          name,\n          ok: true,\n          exact: leaves.every((l) => l.outcome === \"match\"),\n          leaves,\n          issues,\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      } catch (error) {\n        return {\n          name,\n          ok: false,\n          error: error instanceof Error ? error.message : String(error),\n          exact: false,\n          leaves: compareLeaves(fixture.expected, undefined),\n          issues: [],\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      }\n    });\n  });\n\n  const fields = fieldStats(items);\n  const tp = fields.reduce((s, f) => s + f.tp, 0);\n  const fp = fields.reduce((s, f) => s + f.fp, 0);\n  const fn = fields.reduce((s, f) => s + f.fn, 0);\n  const usage = emptyUsage();\n  for (const item of items) {\n    usage.promptTokens += item.usage.promptTokens;\n    usage.completionTokens += item.usage.completionTokens;\n    usage.cacheReadTokens += item.usage.cacheReadTokens;\n    usage.cacheWriteTokens += item.usage.cacheWriteTokens;\n  }\n  const latencies = items.map((i) => i.latencyMs).sort((a, b) => a - b);\n\n  return {\n    schemaId: options.schema.id,\n    mode,\n    ranAt: new Date().toISOString(),\n    items,\n    fields,\n    totals: {\n      fixtures: items.length,\n      ok: items.filter((i) => i.ok).length,\n      exact: items.filter((i) => i.exact).length,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n      usage,\n      calls: items.reduce((s, i) => s + i.calls, 0),\n      ...(prices ? { cost: estimateCost(usage, prices) } : {}),\n      latencyMs: {\n        p50: percentile(latencies, 50),\n        p95: percentile(latencies, 95),\n        max: latencies[latencies.length - 1] ?? 0,\n        total: latencies.reduce((s, l) => s + l, 0),\n      },\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Persistence and diffs\n\nexport function saveReport(report: EvalReport, file: string): void {\n  mkdirSync(dirname(resolve(file)), { recursive: true });\n  writeFileSync(resolve(file), JSON.stringify(report, null, 2) + \"\\n\");\n}\n\nexport function loadReport(file: string): EvalReport | undefined {\n  const path = resolve(file);\n  if (!existsSync(path)) return undefined;\n  return JSON.parse(readFileSync(path, \"utf8\")) as EvalReport;\n}\n\nexport interface FieldDelta {\n  path: string;\n  precision: number | null;\n  recall: number | null;\n}\n\n/** What changed between two runs. Deltas are `next − previous`. */\nexport interface EvalDiff {\n  exact: number;\n  ok: number;\n  precision: number | null;\n  recall: number | null;\n  promptTokens: number;\n  completionTokens: number;\n  cost?: number;\n  p50Ms: number;\n  /** Fields whose precision or recall moved, largest drop first. */\n  fields: FieldDelta[];\n}\n\nfunction delta(a: number | null | undefined, b: number | null | undefined): number | null {\n  return a === null || a === undefined || b === null || b === undefined ? null : b - a;\n}\n\nexport function diffReports(previous: EvalReport, next: EvalReport): EvalDiff {\n  const prevFields = new Map(previous.fields.map((f) => [f.path, f]));\n  const fields: FieldDelta[] = [];\n  for (const field of next.fields) {\n    const before = prevFields.get(field.path);\n    const precision = delta(before?.precision, field.precision);\n    const recall = delta(before?.recall, field.recall);\n    if ((precision !== null && precision !== 0) || (recall !== null && recall !== 0)) {\n      fields.push({ path: field.path, precision, recall });\n    }\n  }\n  fields.sort((a, b) => Math.min(a.precision ?? 0, a.recall ?? 0) - Math.min(b.precision ?? 0, b.recall ?? 0));\n\n  return {\n    exact: next.totals.exact - previous.totals.exact,\n    ok: next.totals.ok - previous.totals.ok,\n    precision: delta(previous.totals.precision, next.totals.precision),\n    recall: delta(previous.totals.recall, next.totals.recall),\n    promptTokens: next.totals.usage.promptTokens - previous.totals.usage.promptTokens,\n    completionTokens: next.totals.usage.completionTokens - previous.totals.usage.completionTokens,\n    ...(next.totals.cost !== undefined && previous.totals.cost !== undefined\n      ? { cost: next.totals.cost - previous.totals.cost }\n      : {}),\n    p50Ms: next.totals.latencyMs.p50 - previous.totals.latencyMs.p50,\n    fields,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Formatting\n\nfunction pct(value: number | null): string {\n  return value === null ? \"  –  \" : `${(value * 100).toFixed(0).padStart(3)}%`;\n}\n\nfunction signed(value: number | null, digits = 0, suffix = \"\"): string {\n  if (value === null || value === 0) return \"\";\n  const text = digits > 0 ? value.toFixed(digits) : String(value);\n  return ` (${value > 0 ? \"+\" : \"\"}${text}${suffix})`;\n}\n\nfunction signedPct(value: number | null): string {\n  return value === null || value === 0 ? \"\" : ` (${value > 0 ? \"+\" : \"\"}${(value * 100).toFixed(0)}pt)`;\n}\n\n/** A plain-text report for a terminal, with deltas when a previous run is given. */\nexport function formatReport(report: EvalReport, diff?: EvalDiff): string {\n  const t = report.totals;\n  const lines: string[] = [];\n  lines.push(\n    `Eval: ${report.schemaId} (${report.mode}) — ${t.fixtures} fixture(s), ${t.ok} ran${signed(diff?.ok ?? null)}, ${t.exact} exact${signed(diff?.exact ?? null)}`,\n  );\n  lines.push(\"\");\n\n  const width = Math.max(5, ...report.fields.map((f) => f.path.length));\n  const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));\n  lines.push(`${\"Field\".padEnd(width)}  Prec  Recall   tp  fp  fn  Δ`);\n  for (const field of report.fields) {\n    const d = deltas.get(field.path);\n    const change = d\n      ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : \"\", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : \"\"]\n          .filter(Boolean)\n          .join(\" \")\n      : \"\";\n    lines.push(\n      `${field.path.padEnd(width)}  ${pct(field.precision)} ${pct(field.recall)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`,\n    );\n  }\n  lines.push(\"\");\n  lines.push(\n    `Overall: precision ${pct(t.precision).trim()}${signedPct(diff?.precision ?? null)}, recall ${pct(t.recall).trim()}${signedPct(diff?.recall ?? null)}`,\n  );\n  const u = t.usage;\n  const cache = u.cacheReadTokens || u.cacheWriteTokens\n    ? `, cache read ${u.cacheReadTokens.toLocaleString(\"en-US\")} / write ${u.cacheWriteTokens.toLocaleString(\"en-US\")}`\n    : \"\";\n  lines.push(\n    `Tokens: ${u.promptTokens.toLocaleString(\"en-US\")} prompt${signed(diff?.promptTokens ?? null)} / ${u.completionTokens.toLocaleString(\"en-US\")} completion${signed(diff?.completionTokens ?? null)}${cache}; ${t.calls} call(s)` +\n      (t.cost !== undefined ? `; cost $${t.cost.toFixed(4)}${signed(diff?.cost ?? null, 4)}` : \"\"),\n  );\n  lines.push(\n    `Latency: p50 ${t.latencyMs.p50}ms${signed(diff?.p50Ms ?? null, 0, \"ms\")}, p95 ${t.latencyMs.p95}ms, max ${t.latencyMs.max}ms`,\n  );\n\n  const failed = report.items.filter((i) => !i.ok);\n  if (failed.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Failed:\");\n    for (const item of failed) lines.push(`  ${item.name}: ${item.error}`);\n  }\n  const imperfect = report.items.filter((i) => i.ok && !i.exact);\n  if (imperfect.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Mismatches:\");\n    for (const item of imperfect) {\n      for (const leaf of item.leaves.filter((l) => l.outcome !== \"match\")) {\n        const detail =\n          leaf.outcome === \"wrong\"\n            ? `expected ${JSON.stringify(leaf.expected)}, got ${JSON.stringify(leaf.actual)}`\n            : leaf.outcome === \"missing\"\n              ? `expected ${JSON.stringify(leaf.expected)}, got nothing`\n              : `unexpected ${JSON.stringify(leaf.actual)}`;\n        const conf = leaf.confidence ? ` [${leaf.confidence}]` : \"\";\n        lines.push(`  ${item.name} › ${leaf.path}: ${detail}${conf}`);\n      }\n    }\n  }\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,WAAW,cAAc,eAAe,YAAY,mBAAmB;AAChF,SAAS,YAAY;AAkBrB,SAAS,OAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,MAAM,MAClC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAOO,SAAS,aAAa,SAAkC;AAC7D,QAAM,WAAW,OAAO;AAAA,IACtB,cAAc,QAAQ;AAAA,IACtB,WAAW,QAAQ;AAAA,IACnB,YAAY,QAAQ;AAAA,EACtB,CAAC;AACD,SAAO,WAAW,QAAQ,EAAE,OAAO,QAAQ,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AACxE;AAEA,SAAS,KAAK,IAAoB;AAChC,SAAO,GAAG,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK;AAC7D;AAGO,SAAS,cAAc,KAAa,SAAkC;AAC3E,SAAO,KAAK,KAAK,GAAG,KAAK,QAAQ,OAAO,EAAE,CAAC,IAAI,aAAa,OAAO,CAAC,OAAO;AAC7E;AAGO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EACzC,YACkB,KACA,KACA,UAChB;AACA;AAAA,MACE,uBAAuB,QAAQ,kBAAkB,GAAG,QAAQ,GAAG;AAAA,IAEjE;AAPgB;AACA;AACA;AAMhB,SAAK,OAAO;AAAA,EACd;AACF;AAWO,IAAM,oBAAN,MAA4C;AAAA,EACjD,YACmB,OACA,KACjB;AAFiB;AACA;AAEjB,cAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,YAAuB;AAAA,MAC3B,KAAK,aAAa,OAAO;AAAA,MACzB,UAAU,QAAQ,OAAO;AAAA,MACzB,SAAS;AAAA,QACP,cAAc,QAAQ;AAAA,QACtB,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,MACA,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,kBAAc,cAAc,KAAK,KAAK,OAAO,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,IAAI,IAAI;AACzF,WAAO;AAAA,EACT;AACF;AAkBO,IAAM,iBAAN,MAAyC;AAAA,EAG9C,YACmB,KACjB,UAAyB,CAAC,GAC1B;AAFiB;AAGjB,SAAK,WAAW,QAAQ,WAAW,IAAI,kBAAkB,QAAQ,UAAU,GAAG,IAAI;AAAA,EACpF;AAAA,EAPiB;AAAA,EASjB,MAAM,SAAS,SAAqD;AAClE,UAAM,OAAO,cAAc,KAAK,KAAK,OAAO;AAC5C,QAAI,WAAW,IAAI,GAAG;AACpB,YAAM,YAAY,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;AACvD,aAAO,UAAU;AAAA,IACnB;AACA,QAAI,KAAK,UAAU;AACjB,aAAO,KAAK,SAAS,SAAS,OAAO;AAAA,IACvC;AACA,UAAM,IAAI,gBAAgB,aAAa,OAAO,GAAG,KAAK,KAAK,QAAQ,OAAO,EAAE;AAAA,EAC9E;AAAA;AAAA,EAGA,OAAe;AACb,QAAI,CAAC,WAAW,KAAK,GAAG,EAAG,QAAO;AAClC,WAAO,YAAY,KAAK,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,EAAE;AAAA,EAClE;AACF;AAOO,SAAS,eAAe,KAAa,MAA2B;AACrE,SAAO,IAAI,eAAe,KAAK,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC,CAAC;AAC/D;;;ACzJA,SAAS,yBAAyB;AAClC,SAAS,eAAAA,cAAa,gBAAAC,eAAc,cAAAC,aAAY,aAAAC,YAAW,iBAAAC,sBAAqB;AAChF,SAAS,UAAU,SAAS,QAAAC,OAAM,eAAe;AACjD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAqHP,SAAS,YAAY,OAAoC;AACvD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAoB,SAAS;AAC7F;AAGA,SAAS,aAAa,OAAgB,SAA8B;AAClE,QAAM,MAAM,CAAC,UAAoC;AAC/C,QAAI,OAAO,UAAU,SAAU,QAAO;AACtC,QAAI,YAAY,KAAK,GAAG;AACtB,YAAM,OAAOJ,cAAa,QAAQ,SAAS,MAAM,IAAI,GAAG,MAAM;AAC9D,aAAO,MAAM,QAAQ,EAAE,OAAO,MAAM,OAAO,KAAK,IAAI,EAAE,KAAK;AAAA,IAC7D;AACA,WAAO;AAAA,EACT;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM,IAAI,CAAC,MAAM;AACtB,YAAM,IAAI,IAAI,CAAC;AACf,aAAO,OAAO,MAAM,WAAW,EAAE,MAAM,EAAE,IAAI;AAAA,IAC/C,CAAC;AAAA,EACH;AACA,SAAO,IAAI,KAAK;AAClB;AAQO,SAAS,aAAa,KAA4B;AACvD,QAAM,OAAO,QAAQ,GAAG;AACxB,MAAI,CAACC,YAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,gCAAgC,IAAI,EAAE;AAAA,EACxD;AACA,QAAM,WAA0B,CAAC;AACjC,QAAM,QAAQF,aAAY,IAAI,EAC3B,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC,EACvD,KAAK;AACR,aAAW,QAAQ,OAAO;AACxB,UAAM,OAAOK,MAAK,MAAM,IAAI;AAC5B,UAAM,SAAS,KAAK,MAAMJ,cAAa,MAAM,MAAM,CAAC;AACpD,UAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AACrD,SAAK,QAAQ,CAAC,KAAK,UAAU;AAC3B,YAAM,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,YAAY,YAAY,EAAE,WAAW,YAAY,EAAE,cAAc,UAAU;AAChG,cAAM,IAAI,MAAM,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,EAAE,0CAA0C;AAAA,MACzG;AACA,YAAM,OAAO,SAAS,MAAM,OAAO;AACnC,eAAS,KAAK;AAAA,QACZ,MAAM,QAAQ,SAAS,KAAK,SAAS,IAAI,GAAG,IAAI,IAAI,KAAK,MAAM;AAAA,QAC/D,OAAO,aAAa,QAAQ,OAAO,QAAQ,IAAI,CAAC;AAAA,QAChD,UAAU,QAAQ;AAAA,MACpB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAKA,SAASK,QAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,IAAI,MAChC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAEA,SAAS,YAAY,OAAyB;AAC5C,SAAO,UAAU,QAAQ,OAAO,UAAU;AAC5C;AAMO,SAAS,cAAc,OAAgB,SAAS,IAA0B;AAC/E,QAAM,SAAS,oBAAI,IAAqB;AACxC,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACrD,WAAO,IAAI,QAAQ,KAAK;AACxB,WAAO;AAAA,EACT;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAgC,GAAG;AAC3E,UAAM,OAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,eAAW,CAAC,UAAU,IAAI,KAAK,cAAc,OAAO,IAAI,GAAG;AACzD,aAAO,IAAI,UAAU,IAAI;AAAA,IAC3B;AAAA,EACF;AACA,SAAO;AACT;AAOO,SAAS,YAAY,GAAY,GAAqB;AAC3D,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,QAAI,EAAE,MAAM,WAAW,KAAK,EAAE,MAAM,WAAW,GAAG;AAChD,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,aAAO,QAAQ,MAAM,CAAC,GAAG,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,IACjD;AAAA,EACF;AACA,SAAOA,QAAO,CAAC,MAAMA,QAAO,CAAC;AAC/B;AAGO,SAAS,cACd,UACA,QACA,aAA8C,CAAC,GACjC;AACd,QAAM,OAAO,cAAc,QAAQ;AACnC,QAAM,MAAM,cAAc,UAAU,CAAC,CAAC;AACtC,QAAM,QAAQ,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,KAAK,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACjE,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,MAAM,KAAK,MAAM,GAAG,EAAE,CAAC;AAC7B,UAAM,aAAa,WAAW,GAAG,GAAG;AACpC,UAAM,SAAqB,EAAE,MAAM,SAAS,QAAQ;AACpD,QAAI,WAAY,QAAO,aAAa;AACpC,QAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG;AACnC,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU,YAAY,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,UAAU;AAAA,IAC1E,WAAW,KAAK,IAAI,IAAI,GAAG;AACzB,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,UAAU;AAAA,IACnB,OAAO;AACL,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU;AAAA,IACnB;AACA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,MAAM,KAAa,KAA4B;AACtD,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAGO,SAAS,WAAW,OAA0C;AACnE,QAAM,SAAS,oBAAI,IAAoD;AACvE,aAAW,QAAQ,OAAO;AACxB,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,QAAQ,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE;AAC7D,cAAQ,KAAK,SAAS;AAAA,QACpB,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,MACJ;AACA,aAAO,IAAI,KAAK,MAAM,KAAK;AAAA,IAC7B;AAAA,EACF;AACA,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EACxB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO;AAAA,IAChC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,IAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,EAC3B,EAAE;AACN;AAEA,SAAS,WAAW,QAA2B,GAAmB;AAChE,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,QAAQ,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,IAAI,CAAC;AAClF,SAAO,OAAO,KAAK,IAAI,GAAG,KAAK,CAAC;AAClC;AAEA,SAAS,aAAoB;AAC3B,SAAO,EAAE,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,kBAAkB,EAAE;AACzF;AAEA,SAAS,SAAS,MAAa,OAAwC;AACrE,MAAI,CAAC,MAAO;AACZ,OAAK,gBAAgB,MAAM;AAC3B,OAAK,oBAAoB,MAAM;AAC/B,OAAK,mBAAmB,MAAM,mBAAmB;AACjD,OAAK,oBAAoB,MAAM,oBAAoB;AACrD;AAGO,SAAS,aAAa,OAAc,QAA6B;AACtE,QAAM,MAAM,CAAC,QAAgB,UAAmB,SAAS,MAAa;AACtE,SACE,IAAI,MAAM,cAAc,OAAO,YAAY,IAC3C,IAAI,MAAM,kBAAkB,OAAO,aAAa,IAChD,IAAI,MAAM,iBAAiB,OAAO,oBAAoB,OAAO,YAAY,IACzE,IAAI,MAAM,kBAAkB,OAAO,qBAAqB,OAAO,YAAY;AAE/E;AAUA,IAAM,UAAU,IAAI,kBAA+B;AAGnD,IAAM,kBAAN,MAA0C;AAAA,EACxC,YAA6B,OAAiB;AAAjB;AAAA,EAAkB;AAAA,EAE/C,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,MAAM,QAAQ,SAAS;AAC7B,QAAI,KAAK;AACP,UAAI,SAAS;AACb,eAAS,IAAI,OAAO,SAAS,KAAK;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AACF;AAEA,eAAe,mBACb,OACA,aACA,IACc;AACd,QAAM,UAAe,IAAI,MAAM,MAAM,MAAM;AAC3C,MAAI,OAAO;AACX,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,YAAY;AACtF,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,QAAQ;AACd,cAAQ,KAAK,IAAI,MAAM,GAAG,MAAM,KAAK,GAAG,KAAK;AAAA,IAC/C;AAAA,EACF,CAAC;AACD,QAAM,QAAQ,IAAI,OAAO;AACzB,SAAO;AACT;AAWA,eAAsB,QAAQ,SAA2C;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,IACb,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA,GAAG;AAAA,EACL,IAAI;AACJ,QAAM,UAAU,IAAI,gBAAgB,QAAQ;AAC5C,QAAM,MAAM,SAAS,WACjB,aAAa,uBAAuB,SACpC,aAAa,8BAA8B;AAE/C,QAAM,QAAQ,MAAM,mBAAmB,UAAU,aAAa,OAAO,SAAS,UAAU;AACtF,UAAM,MAAmB,EAAE,OAAO,WAAW,GAAG,OAAO,EAAE;AACzD,UAAM,OAAO,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AACjD,UAAM,UAAU,YAAY,IAAI;AAChC,WAAO,QAAQ,IAAI,KAAK,YAA+B;AACrD,UAAI;AACF,cAAM,SAAS,MAAM,IAA6B,QAAQ,OAAO;AAAA,UAC/D,GAAG;AAAA,UACH,UAAU;AAAA,QACZ,CAAC;AACD,cAAM,OAAO,aACR,OAA6C,OAC7C;AACL,cAAM,OAAO,aAAc,OAA2D,aAAa,CAAC;AACpG,cAAM,SAAS,aAAc,OAAuC,SAAS,CAAC;AAC9E,cAAM,SAAS,cAAc,QAAQ,UAAU,MAAM,IAAI;AACzD,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,OAAO,MAAM,CAAC,MAAM,EAAE,YAAY,OAAO;AAAA,UAChD;AAAA,UACA;AAAA,UACA,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF,SAAS,OAAO;AACd,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC5D,OAAO;AAAA,UACP,QAAQ,cAAc,QAAQ,UAAU,MAAS;AAAA,UACjD,QAAQ,CAAC;AAAA,UACT,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,SAAS,WAAW,KAAK;AAC/B,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,QAAQ,WAAW;AACzB,aAAW,QAAQ,OAAO;AACxB,UAAM,gBAAgB,KAAK,MAAM;AACjC,UAAM,oBAAoB,KAAK,MAAM;AACrC,UAAM,mBAAmB,KAAK,MAAM;AACpC,UAAM,oBAAoB,KAAK,MAAM;AAAA,EACvC;AACA,QAAM,YAAY,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAEpE,SAAO;AAAA,IACL,UAAU,QAAQ,OAAO;AAAA,IACzB;AAAA,IACA,QAAO,oBAAI,KAAK,GAAE,YAAY;AAAA,IAC9B;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,UAAU,MAAM;AAAA,MAChB,IAAI,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE;AAAA,MAC9B,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAAA,MACpC,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,MACzB;AAAA,MACA,OAAO,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC;AAAA,MAC5C,GAAI,SAAS,EAAE,MAAM,aAAa,OAAO,MAAM,EAAE,IAAI,CAAC;AAAA,MACtD,WAAW;AAAA,QACT,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,UAAU,UAAU,SAAS,CAAC,KAAK;AAAA,QACxC,OAAO,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AACF;AAKO,SAAS,WAAW,QAAoB,MAAoB;AACjE,EAAAH,WAAU,QAAQ,QAAQ,IAAI,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,EAAAC,eAAc,QAAQ,IAAI,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;AACrE;AAEO,SAAS,WAAW,MAAsC;AAC/D,QAAM,OAAO,QAAQ,IAAI;AACzB,MAAI,CAACF,YAAW,IAAI,EAAG,QAAO;AAC9B,SAAO,KAAK,MAAMD,cAAa,MAAM,MAAM,CAAC;AAC9C;AAsBA,SAAS,MAAM,GAA8B,GAA6C;AACxF,SAAO,MAAM,QAAQ,MAAM,UAAa,MAAM,QAAQ,MAAM,SAAY,OAAO,IAAI;AACrF;AAEO,SAAS,YAAY,UAAsB,MAA4B;AAC5E,QAAM,aAAa,IAAI,IAAI,SAAS,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,QAAM,SAAuB,CAAC;AAC9B,aAAW,SAAS,KAAK,QAAQ;AAC/B,UAAM,SAAS,WAAW,IAAI,MAAM,IAAI;AACxC,UAAM,YAAY,MAAM,QAAQ,WAAW,MAAM,SAAS;AAC1D,UAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,MAAM;AACjD,QAAK,cAAc,QAAQ,cAAc,KAAO,WAAW,QAAQ,WAAW,GAAI;AAChF,aAAO,KAAK,EAAE,MAAM,MAAM,MAAM,WAAW,OAAO,CAAC;AAAA,IACrD;AAAA,EACF;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,IAAI,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,CAAC;AAE3G,SAAO;AAAA,IACL,OAAO,KAAK,OAAO,QAAQ,SAAS,OAAO;AAAA,IAC3C,IAAI,KAAK,OAAO,KAAK,SAAS,OAAO;AAAA,IACrC,WAAW,MAAM,SAAS,OAAO,WAAW,KAAK,OAAO,SAAS;AAAA,IACjE,QAAQ,MAAM,SAAS,OAAO,QAAQ,KAAK,OAAO,MAAM;AAAA,IACxD,cAAc,KAAK,OAAO,MAAM,eAAe,SAAS,OAAO,MAAM;AAAA,IACrE,kBAAkB,KAAK,OAAO,MAAM,mBAAmB,SAAS,OAAO,MAAM;AAAA,IAC7E,GAAI,KAAK,OAAO,SAAS,UAAa,SAAS,OAAO,SAAS,SAC3D,EAAE,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO,KAAK,IAChD,CAAC;AAAA,IACL,OAAO,KAAK,OAAO,UAAU,MAAM,SAAS,OAAO,UAAU;AAAA,IAC7D;AAAA,EACF;AACF;AAKA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,eAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAC3E;AAEA,SAAS,OAAO,OAAsB,SAAS,GAAG,SAAS,IAAY;AACrE,MAAI,UAAU,QAAQ,UAAU,EAAG,QAAO;AAC1C,QAAM,OAAO,SAAS,IAAI,MAAM,QAAQ,MAAM,IAAI,OAAO,KAAK;AAC9D,SAAO,KAAK,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,GAAG,MAAM;AAClD;AAEA,SAAS,UAAU,OAA8B;AAC/C,SAAO,UAAU,QAAQ,UAAU,IAAI,KAAK,KAAK,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAClG;AAGO,SAAS,aAAa,QAAoB,MAAyB;AACxE,QAAM,IAAI,OAAO;AACjB,QAAM,QAAkB,CAAC;AACzB,QAAM;AAAA,IACJ,SAAS,OAAO,QAAQ,KAAK,OAAO,IAAI,YAAO,EAAE,QAAQ,gBAAgB,EAAE,EAAE,OAAO,OAAO,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,SAAS,OAAO,MAAM,SAAS,IAAI,CAAC;AAAA,EAC9J;AACA,QAAM,KAAK,EAAE;AAEb,QAAM,QAAQ,KAAK,IAAI,GAAG,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC;AACpE,QAAM,SAAS,IAAI,KAAK,MAAM,UAAU,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnE,QAAM,KAAK,GAAG,QAAQ,OAAO,KAAK,CAAC,qCAAgC;AACnE,aAAW,SAAS,OAAO,QAAQ;AACjC,UAAM,IAAI,OAAO,IAAI,MAAM,IAAI;AAC/B,UAAM,SAAS,IACX,CAAC,EAAE,cAAc,QAAQ,EAAE,cAAc,IAAI,IAAI,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,WAAW,QAAQ,EAAE,WAAW,IAAI,IAAI,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,EAC/J,OAAO,OAAO,EACd,KAAK,GAAG,IACX;AACJ,UAAM;AAAA,MACJ,GAAG,MAAM,KAAK,OAAO,KAAK,CAAC,KAAK,IAAI,MAAM,SAAS,CAAC,IAAI,IAAI,MAAM,MAAM,CAAC,MAAM,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,MAAM;AAAA,IAC1L;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,sBAAsB,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,aAAa,IAAI,CAAC,YAAY,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,UAAU,IAAI,CAAC;AAAA,EACtJ;AACA,QAAM,IAAI,EAAE;AACZ,QAAM,QAAQ,EAAE,mBAAmB,EAAE,mBACjC,gBAAgB,EAAE,gBAAgB,eAAe,OAAO,CAAC,YAAY,EAAE,iBAAiB,eAAe,OAAO,CAAC,KAC/G;AACJ,QAAM;AAAA,IACJ,WAAW,EAAE,aAAa,eAAe,OAAO,CAAC,UAAU,OAAO,MAAM,gBAAgB,IAAI,CAAC,MAAM,EAAE,iBAAiB,eAAe,OAAO,CAAC,cAAc,OAAO,MAAM,oBAAoB,IAAI,CAAC,GAAG,KAAK,KAAK,EAAE,KAAK,cAClN,EAAE,SAAS,SAAY,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAC,GAAG,OAAO,MAAM,QAAQ,MAAM,CAAC,CAAC,KAAK;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,gBAAgB,EAAE,UAAU,GAAG,KAAK,OAAO,MAAM,SAAS,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,EAAE,UAAU,GAAG;AAAA,EAC5H;AAEA,QAAM,SAAS,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE;AAC/C,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,SAAS;AACpB,eAAW,QAAQ,OAAQ,OAAM,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,EAAE;AAAA,EACvE;AACA,QAAM,YAAY,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK;AAC7D,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,aAAa;AACxB,eAAW,QAAQ,WAAW;AAC5B,iBAAW,QAAQ,KAAK,OAAO,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,GAAG;AACnE,cAAM,SACJ,KAAK,YAAY,UACb,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,SAAS,KAAK,UAAU,KAAK,MAAM,CAAC,KAC7E,KAAK,YAAY,YACf,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,kBACzC,cAAc,KAAK,UAAU,KAAK,MAAM,CAAC;AACjD,cAAM,OAAO,KAAK,aAAa,KAAK,KAAK,UAAU,MAAM;AACzD,cAAM,KAAK,KAAK,KAAK,IAAI,WAAM,KAAK,IAAI,KAAK,MAAM,GAAG,IAAI,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":["readdirSync","readFileSync","existsSync","mkdirSync","writeFileSync","join","stable"]}
+{"version":3,"sources":["../src/replay.ts","../src/eval.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Provider, ProviderRequest, ProviderResponse } from \"@sembl/core\";\n\n/** What a recording holds: enough of the request to recognise it, and the answer. */\nexport interface Recording {\n  /** Hash of the parts of the request that determine the answer. */\n  key: string;\n  schemaId: string;\n  request: {\n    systemPrompt: string;\n    userInput: string;\n    jsonSchema: Record<string, unknown>;\n  };\n  response: ProviderResponse;\n  recordedAt: string;\n}\n\n/** JSON with sorted keys, so the same request always hashes the same. */\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_key, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\n/**\n * The key a request records under. Only the parts that reach the model\n * count: the system prompt, the user input and the JSON Schema. The runtime\n * schema and bundle are already folded into those.\n */\nexport function recordingKey(request: ProviderRequest): string {\n  const material = stable({\n    systemPrompt: request.systemPrompt,\n    userInput: request.userInput,\n    jsonSchema: request.jsonSchema,\n  });\n  return createHash(\"sha256\").update(material).digest(\"hex\").slice(0, 24);\n}\n\nfunction slug(id: string): string {\n  return id.replace(/[^A-Za-z0-9_-]+/g, \"_\").slice(0, 40) || \"schema\";\n}\n\n/** Where a recording for a request lives inside a directory. */\nexport function recordingPath(dir: string, request: ProviderRequest): string {\n  return join(dir, `${slug(request.schema.id)}.${recordingKey(request)}.json`);\n}\n\n/** Thrown by {@link ReplayProvider} when no recording matches a request. */\nexport class ReplayMissError extends Error {\n  constructor(\n    public readonly key: string,\n    public readonly dir: string,\n    public readonly schemaId: string,\n  ) {\n    super(\n      `No recording for a \"${schemaId}\" request (key ${key}) in ${dir}. ` +\n        \"Run once with a RecordingProvider, or pass a fallback provider to record misses.\",\n    );\n    this.name = \"ReplayMissError\";\n  }\n}\n\n/**\n * Calls another provider and writes every request/response pair to a\n * directory, one JSON file each, named by schema and request hash. Run your\n * extraction code through it once with real credentials; the files it leaves\n * behind let a {@link ReplayProvider} answer the same requests offline.\n *\n * A request is identified by what reaches the model, so editing a field\n * description or the input produces a new recording rather than a stale hit.\n */\nexport class RecordingProvider implements Provider {\n  constructor(\n    private readonly inner: Provider,\n    private readonly dir: string,\n  ) {\n    mkdirSync(dir, { recursive: true });\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const recording: Recording = {\n      key: recordingKey(request),\n      schemaId: request.schema.id,\n      request: {\n        systemPrompt: request.systemPrompt,\n        userInput: request.userInput,\n        jsonSchema: request.jsonSchema,\n      },\n      response,\n      recordedAt: new Date().toISOString(),\n    };\n    writeFileSync(recordingPath(this.dir, request), JSON.stringify(recording, null, 2) + \"\\n\");\n    return response;\n  }\n}\n\n/** Options for {@link ReplayProvider}. */\nexport interface ReplayOptions {\n  /**\n   * Where to send a request no recording matches. The answer is recorded,\n   * so the next run replays it. Without one, a miss throws\n   * {@link ReplayMissError} — the right behaviour in CI, where a miss means\n   * a fixture changed and nobody re-recorded.\n   */\n  fallback?: Provider;\n}\n\n/**\n * Answers requests from a directory of recordings, never touching the\n * network. Deterministic, free, and fast: the provider to put under tests of\n * your own extraction code.\n */\nexport class ReplayProvider implements Provider {\n  private readonly recorder: RecordingProvider | undefined;\n\n  constructor(\n    private readonly dir: string,\n    options: ReplayOptions = {},\n  ) {\n    this.recorder = options.fallback ? new RecordingProvider(options.fallback, dir) : undefined;\n  }\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const path = recordingPath(this.dir, request);\n    if (existsSync(path)) {\n      const recording = JSON.parse(readFileSync(path, \"utf8\")) as Recording;\n      return recording.response;\n    }\n    if (this.recorder) {\n      return this.recorder.complete(request);\n    }\n    throw new ReplayMissError(recordingKey(request), this.dir, request.schema.id);\n  }\n\n  /** How many recordings the directory holds. */\n  size(): number {\n    if (!existsSync(this.dir)) return 0;\n    return readdirSync(this.dir).filter((f) => f.endsWith(\".json\")).length;\n  }\n}\n\n/**\n * The usual arrangement: replay from `dir`, and record misses through\n * `live` when it is given — say, only when an API key is present — so the\n * same test file works locally with credentials and in CI without them.\n */\nexport function replayOrRecord(dir: string, live?: Provider): Provider {\n  return new ReplayProvider(dir, live ? { fallback: live } : {});\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\";\nimport { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from \"node:fs\";\nimport { basename, dirname, join, resolve } from \"node:path\";\nimport {\n  coerce,\n  partialCoerce,\n  coerceWithProvenance,\n  partialCoerceWithProvenance,\n} from \"@sembl/core\";\nimport type {\n  CoerceInput,\n  CoerceOptions,\n  FieldProvenance,\n  Provider,\n  ProviderRequest,\n  ProviderResponse,\n  ProviderUsage,\n  ResolvedIssue,\n  Source,\n} from \"@sembl/core\";\n\n/** One case: an input and what a correct extraction of it looks like. */\nexport interface EvalFixture {\n  /** Shown in the report. Defaults to the file it was loaded from. */\n  name?: string;\n  input: CoerceInput;\n  /** The expected extraction. Absent and `null` both mean \"not present\". */\n  expected: Record<string, unknown>;\n}\n\n/** Per-million-token prices, for a cost line in the report. */\nexport interface TokenPrices {\n  inputPerMTok: number;\n  outputPerMTok: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache reads. */\n  cacheReadPerMTok?: number;\n  /** Defaults to `inputPerMTok` when the provider reports cache writes. */\n  cacheWritePerMTok?: number;\n}\n\n/** Options for {@link runEval}. Everything in `CoerceOptions` applies per fixture. */\nexport interface EvalOptions extends CoerceOptions {\n  fixtures: readonly EvalFixture[];\n  /** Which coercion to run. Default `\"coerce\"`. */\n  mode?: \"coerce\" | \"partialCoerce\";\n  /** Ask for provenance, so the report can show confidence per field. */\n  provenance?: boolean;\n  /** How many fixtures run at once. Default 1, for stable latency numbers. */\n  concurrency?: number;\n  /** Prices for the cost line. Without them the report has no cost. */\n  prices?: TokenPrices;\n}\n\n/** How one leaf compared. */\nexport type LeafOutcome = \"match\" | \"wrong\" | \"missing\" | \"extra\";\n\nexport interface LeafResult {\n  path: string;\n  outcome: LeafOutcome;\n  expected?: unknown;\n  actual?: unknown;\n  confidence?: FieldProvenance[\"confidence\"];\n}\n\nexport interface EvalItem {\n  name: string;\n  ok: boolean;\n  /** Message of the error the coercion threw, when it did. */\n  error?: string;\n  /** Whether every expected leaf matched and nothing extra came back. */\n  exact: boolean;\n  leaves: LeafResult[];\n  issues: ResolvedIssue[];\n  latencyMs: number;\n  calls: number;\n  usage: Usage;\n}\n\nexport interface Usage {\n  promptTokens: number;\n  completionTokens: number;\n  cacheReadTokens: number;\n  cacheWriteTokens: number;\n}\n\nexport interface FieldStats {\n  path: string;\n  tp: number;\n  fp: number;\n  fn: number;\n  /** `tp / (tp + fp)`; null when nothing was returned for the field. */\n  precision: number | null;\n  /** `tp / (tp + fn)`; null when nothing was expected for the field. */\n  recall: number | null;\n}\n\nexport interface EvalTotals {\n  fixtures: number;\n  ok: number;\n  exact: number;\n  precision: number | null;\n  recall: number | null;\n  usage: Usage;\n  calls: number;\n  cost?: number;\n  latencyMs: { p50: number; p95: number; max: number; total: number };\n}\n\nexport interface EvalReport {\n  schemaId: string;\n  mode: \"coerce\" | \"partialCoerce\";\n  ranAt: string;\n  items: EvalItem[];\n  fields: FieldStats[];\n  totals: EvalTotals;\n}\n\n// ---------------------------------------------------------------------------\n// Fixtures\n\ninterface FileInput {\n  file: string;\n  label?: string;\n}\n\nfunction isFileInput(value: unknown): value is FileInput {\n  return typeof value === \"object\" && value !== null && typeof (value as FileInput).file === \"string\";\n}\n\n/** Resolve `{ file, label }` inputs against the fixture's own directory. */\nfunction resolveInput(input: unknown, baseDir: string): CoerceInput {\n  const one = (value: unknown): Source | string => {\n    if (typeof value === \"string\") return value;\n    if (isFileInput(value)) {\n      const text = readFileSync(resolve(baseDir, value.file), \"utf8\");\n      return value.label ? { label: value.label, text } : { text };\n    }\n    return value as Source;\n  };\n  if (Array.isArray(input)) {\n    return input.map((v) => {\n      const r = one(v);\n      return typeof r === \"string\" ? { text: r } : r;\n    });\n  }\n  return one(input);\n}\n\n/**\n * Load fixtures from a directory: every `*.json` file is either one fixture\n * or an array of them. An input may be `{ \"file\": \"page.html\", \"label\": … }`\n * to pull text from a sibling file, which keeps large scraped pages out of\n * the JSON.\n */\nexport function loadFixtures(dir: string): EvalFixture[] {\n  const root = resolve(dir);\n  if (!existsSync(root)) {\n    throw new Error(`Fixture directory not found: ${root}`);\n  }\n  const fixtures: EvalFixture[] = [];\n  const files = readdirSync(root)\n    .filter((f) => f.endsWith(\".json\") && !f.startsWith(\".\"))\n    .sort();\n  for (const file of files) {\n    const path = join(root, file);\n    const parsed = JSON.parse(readFileSync(path, \"utf8\")) as unknown;\n    const list = Array.isArray(parsed) ? parsed : [parsed];\n    list.forEach((raw, index) => {\n      const fixture = raw as EvalFixture;\n      if (!fixture || typeof fixture !== \"object\" || !(\"input\" in fixture) || !(\"expected\" in fixture)) {\n        throw new Error(`${path}${list.length > 1 ? `[${index}]` : \"\"}: a fixture needs \"input\" and \"expected\"`);\n      }\n      const stem = basename(file, \".json\");\n      fixtures.push({\n        name: fixture.name ?? (list.length > 1 ? `${stem}[${index}]` : stem),\n        input: resolveInput(fixture.input, dirname(path)),\n        expected: fixture.expected,\n      });\n    });\n  }\n  return fixtures;\n}\n\n// ---------------------------------------------------------------------------\n// Scoring\n\nfunction stable(value: unknown): string {\n  return JSON.stringify(value, (_k, v) =>\n    v && typeof v === \"object\" && !Array.isArray(v)\n      ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))\n      : v,\n  );\n}\n\nfunction isPrimitive(value: unknown): boolean {\n  return value === null || typeof value !== \"object\";\n}\n\n/**\n * Flatten a value to its leaves, keyed by dotted path. Objects recurse;\n * arrays are leaves. `null` and `undefined` are absence, not leaves.\n */\nexport function flattenLeaves(value: unknown, prefix = \"\"): Map<string, unknown> {\n  const leaves = new Map<string, unknown>();\n  if (value === null || value === undefined) return leaves;\n  if (typeof value !== \"object\" || Array.isArray(value)) {\n    leaves.set(prefix, value);\n    return leaves;\n  }\n  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {\n    const path = prefix ? `${prefix}.${key}` : key;\n    for (const [leafPath, leaf] of flattenLeaves(child, path)) {\n      leaves.set(leafPath, leaf);\n    }\n  }\n  return leaves;\n}\n\n/**\n * Whether two leaves agree. Arrays of primitives compare as multisets —\n * the order amenities come back in is not a fact about the listing —\n * and everything else compares structurally.\n */\nexport function leavesEqual(a: unknown, b: unknown): boolean {\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    if (a.every(isPrimitive) && b.every(isPrimitive)) {\n      const sortedA = [...a].map(String).sort();\n      const sortedB = [...b].map(String).sort();\n      return sortedA.every((v, i) => v === sortedB[i]);\n    }\n  }\n  return stable(a) === stable(b);\n}\n\n/** Compare one extraction against its expectation, leaf by leaf. */\nexport function compareLeaves(\n  expected: Record<string, unknown>,\n  actual: Record<string, unknown> | undefined,\n  provenance: Record<string, FieldProvenance> = {},\n): LeafResult[] {\n  const want = flattenLeaves(expected);\n  const got = flattenLeaves(actual ?? {});\n  const paths = [...new Set([...want.keys(), ...got.keys()])].sort();\n  return paths.map((path) => {\n    const top = path.split(\".\")[0];\n    const confidence = provenance[top]?.confidence;\n    const result: LeafResult = { path, outcome: \"match\" };\n    if (confidence) result.confidence = confidence;\n    if (want.has(path) && got.has(path)) {\n      result.expected = want.get(path);\n      result.actual = got.get(path);\n      result.outcome = leavesEqual(want.get(path), got.get(path)) ? \"match\" : \"wrong\";\n    } else if (want.has(path)) {\n      result.expected = want.get(path);\n      result.outcome = \"missing\";\n    } else {\n      result.actual = got.get(path);\n      result.outcome = \"extra\";\n    }\n    return result;\n  });\n}\n\nfunction ratio(num: number, den: number): number | null {\n  return den === 0 ? null : num / den;\n}\n\n/** Aggregate leaf outcomes into per-field precision and recall. */\nexport function fieldStats(items: readonly EvalItem[]): FieldStats[] {\n  const byPath = new Map<string, { tp: number; fp: number; fn: number }>();\n  for (const item of items) {\n    for (const leaf of item.leaves) {\n      const stats = byPath.get(leaf.path) ?? { tp: 0, fp: 0, fn: 0 };\n      switch (leaf.outcome) {\n        case \"match\":\n          stats.tp += 1;\n          break;\n        case \"wrong\":\n          stats.fp += 1;\n          stats.fn += 1;\n          break;\n        case \"missing\":\n          stats.fn += 1;\n          break;\n        case \"extra\":\n          stats.fp += 1;\n          break;\n      }\n      byPath.set(leaf.path, stats);\n    }\n  }\n  return [...byPath.entries()]\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([path, { tp, fp, fn }]) => ({\n      path,\n      tp,\n      fp,\n      fn,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n    }));\n}\n\nfunction percentile(sorted: readonly number[], p: number): number {\n  if (sorted.length === 0) return 0;\n  const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);\n  return sorted[Math.max(0, index)];\n}\n\nfunction emptyUsage(): Usage {\n  return { promptTokens: 0, completionTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };\n}\n\nfunction addUsage(into: Usage, usage: ProviderUsage | undefined): void {\n  if (!usage) return;\n  into.promptTokens += usage.promptTokens;\n  into.completionTokens += usage.completionTokens;\n  into.cacheReadTokens += usage.cacheReadTokens ?? 0;\n  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;\n}\n\n/** Dollars, from per-million-token prices. */\nexport function estimateCost(usage: Usage, prices: TokenPrices): number {\n  const per = (tokens: number, price: number) => (tokens / 1_000_000) * price;\n  return (\n    per(usage.promptTokens, prices.inputPerMTok) +\n    per(usage.completionTokens, prices.outputPerMTok) +\n    per(usage.cacheReadTokens, prices.cacheReadPerMTok ?? prices.inputPerMTok) +\n    per(usage.cacheWriteTokens, prices.cacheWritePerMTok ?? prices.inputPerMTok)\n  );\n}\n\n// ---------------------------------------------------------------------------\n// Running\n\ninterface ItemContext {\n  usage: Usage;\n  calls: number;\n}\n\nconst context = new AsyncLocalStorage<ItemContext>();\n\n/** Credits each call's usage to whichever fixture is running it. */\nclass MeteredProvider implements Provider {\n  constructor(private readonly inner: Provider) {}\n\n  async complete(request: ProviderRequest): Promise<ProviderResponse> {\n    const response = await this.inner.complete(request);\n    const ctx = context.getStore();\n    if (ctx) {\n      ctx.calls += 1;\n      addUsage(ctx.usage, response.usage);\n    }\n    return response;\n  }\n}\n\nasync function mapWithConcurrency<T, R>(\n  items: readonly T[],\n  concurrency: number,\n  fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results: R[] = new Array(items.length);\n  let next = 0;\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {\n    while (next < items.length) {\n      const index = next++;\n      results[index] = await fn(items[index], index);\n    }\n  });\n  await Promise.all(workers);\n  return results;\n}\n\n/**\n * Run every fixture through a coercion and score the results.\n *\n * Per-field precision and recall make a description change measurable: a\n * field whose recall dropped after a rewording is a field the rewording\n * hurt. Token usage and latency come along so the cost of a change is\n * visible next to its effect. Pair it with a `ReplayProvider` to run in CI\n * without spend.\n */\nexport async function runEval(options: EvalOptions): Promise<EvalReport> {\n  const {\n    fixtures,\n    mode = \"coerce\",\n    provenance = false,\n    concurrency = 1,\n    prices,\n    provider,\n    ...coerceOptions\n  } = options;\n  const metered = new MeteredProvider(provider);\n  const run = mode === \"coerce\"\n    ? provenance ? coerceWithProvenance : coerce\n    : provenance ? partialCoerceWithProvenance : partialCoerce;\n\n  const items = await mapWithConcurrency(fixtures, concurrency, async (fixture, index) => {\n    const ctx: ItemContext = { usage: emptyUsage(), calls: 0 };\n    const name = fixture.name ?? `fixture ${index + 1}`;\n    const started = performance.now();\n    return context.run(ctx, async (): Promise<EvalItem> => {\n      try {\n        const result = await run<Record<string, unknown>>(fixture.input, {\n          ...coerceOptions,\n          provider: metered,\n        });\n        const data = provenance\n          ? (result as { data: Record<string, unknown> }).data\n          : (result as Record<string, unknown>);\n        const prov = provenance ? (result as { provenance: Record<string, FieldProvenance> }).provenance : {};\n        const issues = provenance ? (result as { issues: ResolvedIssue[] }).issues : [];\n        const leaves = compareLeaves(fixture.expected, data, prov);\n        return {\n          name,\n          ok: true,\n          exact: leaves.every((l) => l.outcome === \"match\"),\n          leaves,\n          issues,\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      } catch (error) {\n        return {\n          name,\n          ok: false,\n          error: error instanceof Error ? error.message : String(error),\n          exact: false,\n          leaves: compareLeaves(fixture.expected, undefined),\n          issues: [],\n          latencyMs: Math.round(performance.now() - started),\n          calls: ctx.calls,\n          usage: ctx.usage,\n        };\n      }\n    });\n  });\n\n  const fields = fieldStats(items);\n  const tp = fields.reduce((s, f) => s + f.tp, 0);\n  const fp = fields.reduce((s, f) => s + f.fp, 0);\n  const fn = fields.reduce((s, f) => s + f.fn, 0);\n  const usage = emptyUsage();\n  for (const item of items) {\n    usage.promptTokens += item.usage.promptTokens;\n    usage.completionTokens += item.usage.completionTokens;\n    usage.cacheReadTokens += item.usage.cacheReadTokens;\n    usage.cacheWriteTokens += item.usage.cacheWriteTokens;\n  }\n  const latencies = items.map((i) => i.latencyMs).sort((a, b) => a - b);\n\n  return {\n    schemaId: options.schema.id,\n    mode,\n    ranAt: new Date().toISOString(),\n    items,\n    fields,\n    totals: {\n      fixtures: items.length,\n      ok: items.filter((i) => i.ok).length,\n      exact: items.filter((i) => i.exact).length,\n      precision: ratio(tp, tp + fp),\n      recall: ratio(tp, tp + fn),\n      usage,\n      calls: items.reduce((s, i) => s + i.calls, 0),\n      ...(prices ? { cost: estimateCost(usage, prices) } : {}),\n      latencyMs: {\n        p50: percentile(latencies, 50),\n        p95: percentile(latencies, 95),\n        max: latencies[latencies.length - 1] ?? 0,\n        total: latencies.reduce((s, l) => s + l, 0),\n      },\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Persistence and diffs\n\nexport function saveReport(report: EvalReport, file: string): void {\n  mkdirSync(dirname(resolve(file)), { recursive: true });\n  writeFileSync(resolve(file), JSON.stringify(report, null, 2) + \"\\n\");\n}\n\nexport function loadReport(file: string): EvalReport | undefined {\n  const path = resolve(file);\n  if (!existsSync(path)) return undefined;\n  return JSON.parse(readFileSync(path, \"utf8\")) as EvalReport;\n}\n\nexport interface FieldDelta {\n  path: string;\n  precision: number | null;\n  recall: number | null;\n}\n\n/** What changed between two runs. Deltas are `next − previous`. */\nexport interface EvalDiff {\n  exact: number;\n  ok: number;\n  precision: number | null;\n  recall: number | null;\n  promptTokens: number;\n  completionTokens: number;\n  cost?: number;\n  p50Ms: number;\n  /** Fields whose precision or recall moved, largest drop first. */\n  fields: FieldDelta[];\n}\n\nfunction delta(a: number | null | undefined, b: number | null | undefined): number | null {\n  return a === null || a === undefined || b === null || b === undefined ? null : b - a;\n}\n\nexport function diffReports(previous: EvalReport, next: EvalReport): EvalDiff {\n  const prevFields = new Map(previous.fields.map((f) => [f.path, f]));\n  const fields: FieldDelta[] = [];\n  for (const field of next.fields) {\n    const before = prevFields.get(field.path);\n    const precision = delta(before?.precision, field.precision);\n    const recall = delta(before?.recall, field.recall);\n    if ((precision !== null && precision !== 0) || (recall !== null && recall !== 0)) {\n      fields.push({ path: field.path, precision, recall });\n    }\n  }\n  fields.sort((a, b) => Math.min(a.precision ?? 0, a.recall ?? 0) - Math.min(b.precision ?? 0, b.recall ?? 0));\n\n  return {\n    exact: next.totals.exact - previous.totals.exact,\n    ok: next.totals.ok - previous.totals.ok,\n    precision: delta(previous.totals.precision, next.totals.precision),\n    recall: delta(previous.totals.recall, next.totals.recall),\n    promptTokens: next.totals.usage.promptTokens - previous.totals.usage.promptTokens,\n    completionTokens: next.totals.usage.completionTokens - previous.totals.usage.completionTokens,\n    ...(next.totals.cost !== undefined && previous.totals.cost !== undefined\n      ? { cost: next.totals.cost - previous.totals.cost }\n      : {}),\n    p50Ms: next.totals.latencyMs.p50 - previous.totals.latencyMs.p50,\n    fields,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Formatting\n\nfunction pct(value: number | null): string {\n  return value === null ? \"  –  \" : `${(value * 100).toFixed(0).padStart(3)}%`;\n}\n\nfunction signed(value: number | null, digits = 0, suffix = \"\"): string {\n  if (value === null || value === 0) return \"\";\n  const text = digits > 0 ? value.toFixed(digits) : String(value);\n  return ` (${value > 0 ? \"+\" : \"\"}${text}${suffix})`;\n}\n\nfunction signedPct(value: number | null): string {\n  return value === null || value === 0 ? \"\" : ` (${value > 0 ? \"+\" : \"\"}${(value * 100).toFixed(0)}pt)`;\n}\n\n/** A plain-text report for a terminal, with deltas when a previous run is given. */\nexport function formatReport(report: EvalReport, diff?: EvalDiff): string {\n  const t = report.totals;\n  const lines: string[] = [];\n  lines.push(\n    `Eval: ${report.schemaId} (${report.mode}) — ${t.fixtures} fixture(s), ${t.ok} ran${signed(diff?.ok ?? null)}, ${t.exact} exact${signed(diff?.exact ?? null)}`,\n  );\n  lines.push(\"\");\n\n  const width = Math.max(5, ...report.fields.map((f) => f.path.length));\n  const deltas = new Map((diff?.fields ?? []).map((f) => [f.path, f]));\n  lines.push(`${\"Field\".padEnd(width)}  Prec  Recall   tp  fp  fn  Δ`);\n  const col = (value: number | null, w: number) => pct(value).trim().padStart(w);\n  for (const field of report.fields) {\n    const d = deltas.get(field.path);\n    const change = d\n      ? [d.precision !== null && d.precision !== 0 ? `P${signedPct(d.precision).trim()}` : \"\", d.recall !== null && d.recall !== 0 ? `R${signedPct(d.recall).trim()}` : \"\"]\n          .filter(Boolean)\n          .join(\" \")\n      : \"\";\n    lines.push(\n      `${field.path.padEnd(width)}  ${col(field.precision, 4)}  ${col(field.recall, 6)}   ${String(field.tp).padStart(2)}  ${String(field.fp).padStart(2)}  ${String(field.fn).padStart(2)}  ${change}`,\n    );\n  }\n  lines.push(\"\");\n  lines.push(\n    `Overall: precision ${pct(t.precision).trim()}${signedPct(diff?.precision ?? null)}, recall ${pct(t.recall).trim()}${signedPct(diff?.recall ?? null)}`,\n  );\n  const u = t.usage;\n  const cache = u.cacheReadTokens || u.cacheWriteTokens\n    ? `, cache read ${u.cacheReadTokens.toLocaleString(\"en-US\")} / write ${u.cacheWriteTokens.toLocaleString(\"en-US\")}`\n    : \"\";\n  lines.push(\n    `Tokens: ${u.promptTokens.toLocaleString(\"en-US\")} prompt${signed(diff?.promptTokens ?? null)} / ${u.completionTokens.toLocaleString(\"en-US\")} completion${signed(diff?.completionTokens ?? null)}${cache}; ${t.calls} call(s)` +\n      (t.cost !== undefined ? `; cost $${t.cost.toFixed(4)}${signed(diff?.cost ?? null, 4)}` : \"\"),\n  );\n  lines.push(\n    `Latency: p50 ${t.latencyMs.p50}ms${signed(diff?.p50Ms ?? null, 0, \"ms\")}, p95 ${t.latencyMs.p95}ms, max ${t.latencyMs.max}ms`,\n  );\n\n  const failed = report.items.filter((i) => !i.ok);\n  if (failed.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Failed:\");\n    for (const item of failed) lines.push(`  ${item.name}: ${item.error}`);\n  }\n  const imperfect = report.items.filter((i) => i.ok && !i.exact);\n  if (imperfect.length > 0) {\n    lines.push(\"\");\n    lines.push(\"Mismatches:\");\n    for (const item of imperfect) {\n      for (const leaf of item.leaves.filter((l) => l.outcome !== \"match\")) {\n        const detail =\n          leaf.outcome === \"wrong\"\n            ? `expected ${JSON.stringify(leaf.expected)}, got ${JSON.stringify(leaf.actual)}`\n            : leaf.outcome === \"missing\"\n              ? `expected ${JSON.stringify(leaf.expected)}, got nothing`\n              : `unexpected ${JSON.stringify(leaf.actual)}`;\n        const conf = leaf.confidence ? ` [${leaf.confidence}]` : \"\";\n        lines.push(`  ${item.name} › ${leaf.path}: ${detail}${conf}`);\n      }\n    }\n  }\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,WAAW,cAAc,eAAe,YAAY,mBAAmB;AAChF,SAAS,YAAY;AAkBrB,SAAS,OAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,MAAM,MAClC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAOO,SAAS,aAAa,SAAkC;AAC7D,QAAM,WAAW,OAAO;AAAA,IACtB,cAAc,QAAQ;AAAA,IACtB,WAAW,QAAQ;AAAA,IACnB,YAAY,QAAQ;AAAA,EACtB,CAAC;AACD,SAAO,WAAW,QAAQ,EAAE,OAAO,QAAQ,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AACxE;AAEA,SAAS,KAAK,IAAoB;AAChC,SAAO,GAAG,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK;AAC7D;AAGO,SAAS,cAAc,KAAa,SAAkC;AAC3E,SAAO,KAAK,KAAK,GAAG,KAAK,QAAQ,OAAO,EAAE,CAAC,IAAI,aAAa,OAAO,CAAC,OAAO;AAC7E;AAGO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EACzC,YACkB,KACA,KACA,UAChB;AACA;AAAA,MACE,uBAAuB,QAAQ,kBAAkB,GAAG,QAAQ,GAAG;AAAA,IAEjE;AAPgB;AACA;AACA;AAMhB,SAAK,OAAO;AAAA,EACd;AACF;AAWO,IAAM,oBAAN,MAA4C;AAAA,EACjD,YACmB,OACA,KACjB;AAFiB;AACA;AAEjB,cAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,YAAuB;AAAA,MAC3B,KAAK,aAAa,OAAO;AAAA,MACzB,UAAU,QAAQ,OAAO;AAAA,MACzB,SAAS;AAAA,QACP,cAAc,QAAQ;AAAA,QACtB,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,MACA,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,kBAAc,cAAc,KAAK,KAAK,OAAO,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,IAAI,IAAI;AACzF,WAAO;AAAA,EACT;AACF;AAkBO,IAAM,iBAAN,MAAyC;AAAA,EAG9C,YACmB,KACjB,UAAyB,CAAC,GAC1B;AAFiB;AAGjB,SAAK,WAAW,QAAQ,WAAW,IAAI,kBAAkB,QAAQ,UAAU,GAAG,IAAI;AAAA,EACpF;AAAA,EAPiB;AAAA,EASjB,MAAM,SAAS,SAAqD;AAClE,UAAM,OAAO,cAAc,KAAK,KAAK,OAAO;AAC5C,QAAI,WAAW,IAAI,GAAG;AACpB,YAAM,YAAY,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;AACvD,aAAO,UAAU;AAAA,IACnB;AACA,QAAI,KAAK,UAAU;AACjB,aAAO,KAAK,SAAS,SAAS,OAAO;AAAA,IACvC;AACA,UAAM,IAAI,gBAAgB,aAAa,OAAO,GAAG,KAAK,KAAK,QAAQ,OAAO,EAAE;AAAA,EAC9E;AAAA;AAAA,EAGA,OAAe;AACb,QAAI,CAAC,WAAW,KAAK,GAAG,EAAG,QAAO;AAClC,WAAO,YAAY,KAAK,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,EAAE;AAAA,EAClE;AACF;AAOO,SAAS,eAAe,KAAa,MAA2B;AACrE,SAAO,IAAI,eAAe,KAAK,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC,CAAC;AAC/D;;;ACzJA,SAAS,yBAAyB;AAClC,SAAS,eAAAA,cAAa,gBAAAC,eAAc,cAAAC,aAAY,aAAAC,YAAW,iBAAAC,sBAAqB;AAChF,SAAS,UAAU,SAAS,QAAAC,OAAM,eAAe;AACjD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAqHP,SAAS,YAAY,OAAoC;AACvD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAoB,SAAS;AAC7F;AAGA,SAAS,aAAa,OAAgB,SAA8B;AAClE,QAAM,MAAM,CAAC,UAAoC;AAC/C,QAAI,OAAO,UAAU,SAAU,QAAO;AACtC,QAAI,YAAY,KAAK,GAAG;AACtB,YAAM,OAAOJ,cAAa,QAAQ,SAAS,MAAM,IAAI,GAAG,MAAM;AAC9D,aAAO,MAAM,QAAQ,EAAE,OAAO,MAAM,OAAO,KAAK,IAAI,EAAE,KAAK;AAAA,IAC7D;AACA,WAAO;AAAA,EACT;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM,IAAI,CAAC,MAAM;AACtB,YAAM,IAAI,IAAI,CAAC;AACf,aAAO,OAAO,MAAM,WAAW,EAAE,MAAM,EAAE,IAAI;AAAA,IAC/C,CAAC;AAAA,EACH;AACA,SAAO,IAAI,KAAK;AAClB;AAQO,SAAS,aAAa,KAA4B;AACvD,QAAM,OAAO,QAAQ,GAAG;AACxB,MAAI,CAACC,YAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,gCAAgC,IAAI,EAAE;AAAA,EACxD;AACA,QAAM,WAA0B,CAAC;AACjC,QAAM,QAAQF,aAAY,IAAI,EAC3B,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC,EACvD,KAAK;AACR,aAAW,QAAQ,OAAO;AACxB,UAAM,OAAOK,MAAK,MAAM,IAAI;AAC5B,UAAM,SAAS,KAAK,MAAMJ,cAAa,MAAM,MAAM,CAAC;AACpD,UAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AACrD,SAAK,QAAQ,CAAC,KAAK,UAAU;AAC3B,YAAM,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,YAAY,YAAY,EAAE,WAAW,YAAY,EAAE,cAAc,UAAU;AAChG,cAAM,IAAI,MAAM,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,EAAE,0CAA0C;AAAA,MACzG;AACA,YAAM,OAAO,SAAS,MAAM,OAAO;AACnC,eAAS,KAAK;AAAA,QACZ,MAAM,QAAQ,SAAS,KAAK,SAAS,IAAI,GAAG,IAAI,IAAI,KAAK,MAAM;AAAA,QAC/D,OAAO,aAAa,QAAQ,OAAO,QAAQ,IAAI,CAAC;AAAA,QAChD,UAAU,QAAQ;AAAA,MACpB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAKA,SAASK,QAAO,OAAwB;AACtC,SAAO,KAAK;AAAA,IAAU;AAAA,IAAO,CAAC,IAAI,MAChC,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,CAAC,IAC1C,OAAO,YAAY,OAAO,QAAQ,CAA4B,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,IAChH;AAAA,EACN;AACF;AAEA,SAAS,YAAY,OAAyB;AAC5C,SAAO,UAAU,QAAQ,OAAO,UAAU;AAC5C;AAMO,SAAS,cAAc,OAAgB,SAAS,IAA0B;AAC/E,QAAM,SAAS,oBAAI,IAAqB;AACxC,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACrD,WAAO,IAAI,QAAQ,KAAK;AACxB,WAAO;AAAA,EACT;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAgC,GAAG;AAC3E,UAAM,OAAO,SAAS,GAAG,MAAM,IAAI,GAAG,KAAK;AAC3C,eAAW,CAAC,UAAU,IAAI,KAAK,cAAc,OAAO,IAAI,GAAG;AACzD,aAAO,IAAI,UAAU,IAAI;AAAA,IAC3B;AAAA,EACF;AACA,SAAO;AACT;AAOO,SAAS,YAAY,GAAY,GAAqB;AAC3D,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,QAAI,EAAE,MAAM,WAAW,KAAK,EAAE,MAAM,WAAW,GAAG;AAChD,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,YAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,EAAE,KAAK;AACxC,aAAO,QAAQ,MAAM,CAAC,GAAG,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,IACjD;AAAA,EACF;AACA,SAAOA,QAAO,CAAC,MAAMA,QAAO,CAAC;AAC/B;AAGO,SAAS,cACd,UACA,QACA,aAA8C,CAAC,GACjC;AACd,QAAM,OAAO,cAAc,QAAQ;AACnC,QAAM,MAAM,cAAc,UAAU,CAAC,CAAC;AACtC,QAAM,QAAQ,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,KAAK,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACjE,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,MAAM,KAAK,MAAM,GAAG,EAAE,CAAC;AAC7B,UAAM,aAAa,WAAW,GAAG,GAAG;AACpC,UAAM,SAAqB,EAAE,MAAM,SAAS,QAAQ;AACpD,QAAI,WAAY,QAAO,aAAa;AACpC,QAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG;AACnC,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU,YAAY,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,UAAU;AAAA,IAC1E,WAAW,KAAK,IAAI,IAAI,GAAG;AACzB,aAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,aAAO,UAAU;AAAA,IACnB,OAAO;AACL,aAAO,SAAS,IAAI,IAAI,IAAI;AAC5B,aAAO,UAAU;AAAA,IACnB;AACA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,MAAM,KAAa,KAA4B;AACtD,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAGO,SAAS,WAAW,OAA0C;AACnE,QAAM,SAAS,oBAAI,IAAoD;AACvE,aAAW,QAAQ,OAAO;AACxB,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,QAAQ,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE;AAC7D,cAAQ,KAAK,SAAS;AAAA,QACpB,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,QACF,KAAK;AACH,gBAAM,MAAM;AACZ;AAAA,MACJ;AACA,aAAO,IAAI,KAAK,MAAM,KAAK;AAAA,IAC7B;AAAA,EACF;AACA,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EACxB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO;AAAA,IAChC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,IAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,EAC3B,EAAE;AACN;AAEA,SAAS,WAAW,QAA2B,GAAmB;AAChE,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,QAAQ,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,IAAI,CAAC;AAClF,SAAO,OAAO,KAAK,IAAI,GAAG,KAAK,CAAC;AAClC;AAEA,SAAS,aAAoB;AAC3B,SAAO,EAAE,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,kBAAkB,EAAE;AACzF;AAEA,SAAS,SAAS,MAAa,OAAwC;AACrE,MAAI,CAAC,MAAO;AACZ,OAAK,gBAAgB,MAAM;AAC3B,OAAK,oBAAoB,MAAM;AAC/B,OAAK,mBAAmB,MAAM,mBAAmB;AACjD,OAAK,oBAAoB,MAAM,oBAAoB;AACrD;AAGO,SAAS,aAAa,OAAc,QAA6B;AACtE,QAAM,MAAM,CAAC,QAAgB,UAAmB,SAAS,MAAa;AACtE,SACE,IAAI,MAAM,cAAc,OAAO,YAAY,IAC3C,IAAI,MAAM,kBAAkB,OAAO,aAAa,IAChD,IAAI,MAAM,iBAAiB,OAAO,oBAAoB,OAAO,YAAY,IACzE,IAAI,MAAM,kBAAkB,OAAO,qBAAqB,OAAO,YAAY;AAE/E;AAUA,IAAM,UAAU,IAAI,kBAA+B;AAGnD,IAAM,kBAAN,MAA0C;AAAA,EACxC,YAA6B,OAAiB;AAAjB;AAAA,EAAkB;AAAA,EAE/C,MAAM,SAAS,SAAqD;AAClE,UAAM,WAAW,MAAM,KAAK,MAAM,SAAS,OAAO;AAClD,UAAM,MAAM,QAAQ,SAAS;AAC7B,QAAI,KAAK;AACP,UAAI,SAAS;AACb,eAAS,IAAI,OAAO,SAAS,KAAK;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AACF;AAEA,eAAe,mBACb,OACA,aACA,IACc;AACd,QAAM,UAAe,IAAI,MAAM,MAAM,MAAM;AAC3C,MAAI,OAAO;AACX,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,YAAY;AACtF,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,QAAQ;AACd,cAAQ,KAAK,IAAI,MAAM,GAAG,MAAM,KAAK,GAAG,KAAK;AAAA,IAC/C;AAAA,EACF,CAAC;AACD,QAAM,QAAQ,IAAI,OAAO;AACzB,SAAO;AACT;AAWA,eAAsB,QAAQ,SAA2C;AACvE,QAAM;AAAA,IACJ;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,IACb,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA,GAAG;AAAA,EACL,IAAI;AACJ,QAAM,UAAU,IAAI,gBAAgB,QAAQ;AAC5C,QAAM,MAAM,SAAS,WACjB,aAAa,uBAAuB,SACpC,aAAa,8BAA8B;AAE/C,QAAM,QAAQ,MAAM,mBAAmB,UAAU,aAAa,OAAO,SAAS,UAAU;AACtF,UAAM,MAAmB,EAAE,OAAO,WAAW,GAAG,OAAO,EAAE;AACzD,UAAM,OAAO,QAAQ,QAAQ,WAAW,QAAQ,CAAC;AACjD,UAAM,UAAU,YAAY,IAAI;AAChC,WAAO,QAAQ,IAAI,KAAK,YAA+B;AACrD,UAAI;AACF,cAAM,SAAS,MAAM,IAA6B,QAAQ,OAAO;AAAA,UAC/D,GAAG;AAAA,UACH,UAAU;AAAA,QACZ,CAAC;AACD,cAAM,OAAO,aACR,OAA6C,OAC7C;AACL,cAAM,OAAO,aAAc,OAA2D,aAAa,CAAC;AACpG,cAAM,SAAS,aAAc,OAAuC,SAAS,CAAC;AAC9E,cAAM,SAAS,cAAc,QAAQ,UAAU,MAAM,IAAI;AACzD,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,OAAO,MAAM,CAAC,MAAM,EAAE,YAAY,OAAO;AAAA,UAChD;AAAA,UACA;AAAA,UACA,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF,SAAS,OAAO;AACd,eAAO;AAAA,UACL;AAAA,UACA,IAAI;AAAA,UACJ,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC5D,OAAO;AAAA,UACP,QAAQ,cAAc,QAAQ,UAAU,MAAS;AAAA,UACjD,QAAQ,CAAC;AAAA,UACT,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AAAA,UACjD,OAAO,IAAI;AAAA,UACX,OAAO,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,QAAM,SAAS,WAAW,KAAK;AAC/B,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,IAAI,CAAC;AAC9C,QAAM,QAAQ,WAAW;AACzB,aAAW,QAAQ,OAAO;AACxB,UAAM,gBAAgB,KAAK,MAAM;AACjC,UAAM,oBAAoB,KAAK,MAAM;AACrC,UAAM,mBAAmB,KAAK,MAAM;AACpC,UAAM,oBAAoB,KAAK,MAAM;AAAA,EACvC;AACA,QAAM,YAAY,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAEpE,SAAO;AAAA,IACL,UAAU,QAAQ,OAAO;AAAA,IACzB;AAAA,IACA,QAAO,oBAAI,KAAK,GAAE,YAAY;AAAA,IAC9B;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,UAAU,MAAM;AAAA,MAChB,IAAI,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE;AAAA,MAC9B,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAAA,MACpC,WAAW,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5B,QAAQ,MAAM,IAAI,KAAK,EAAE;AAAA,MACzB;AAAA,MACA,OAAO,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC;AAAA,MAC5C,GAAI,SAAS,EAAE,MAAM,aAAa,OAAO,MAAM,EAAE,IAAI,CAAC;AAAA,MACtD,WAAW;AAAA,QACT,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,WAAW,WAAW,EAAE;AAAA,QAC7B,KAAK,UAAU,UAAU,SAAS,CAAC,KAAK;AAAA,QACxC,OAAO,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AACF;AAKO,SAAS,WAAW,QAAoB,MAAoB;AACjE,EAAAH,WAAU,QAAQ,QAAQ,IAAI,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,EAAAC,eAAc,QAAQ,IAAI,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;AACrE;AAEO,SAAS,WAAW,MAAsC;AAC/D,QAAM,OAAO,QAAQ,IAAI;AACzB,MAAI,CAACF,YAAW,IAAI,EAAG,QAAO;AAC9B,SAAO,KAAK,MAAMD,cAAa,MAAM,MAAM,CAAC;AAC9C;AAsBA,SAAS,MAAM,GAA8B,GAA6C;AACxF,SAAO,MAAM,QAAQ,MAAM,UAAa,MAAM,QAAQ,MAAM,SAAY,OAAO,IAAI;AACrF;AAEO,SAAS,YAAY,UAAsB,MAA4B;AAC5E,QAAM,aAAa,IAAI,IAAI,SAAS,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,QAAM,SAAuB,CAAC;AAC9B,aAAW,SAAS,KAAK,QAAQ;AAC/B,UAAM,SAAS,WAAW,IAAI,MAAM,IAAI;AACxC,UAAM,YAAY,MAAM,QAAQ,WAAW,MAAM,SAAS;AAC1D,UAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,MAAM;AACjD,QAAK,cAAc,QAAQ,cAAc,KAAO,WAAW,QAAQ,WAAW,GAAI;AAChF,aAAO,KAAK,EAAE,MAAM,MAAM,MAAM,WAAW,OAAO,CAAC;AAAA,IACrD;AAAA,EACF;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,IAAI,KAAK,IAAI,EAAE,aAAa,GAAG,EAAE,UAAU,CAAC,CAAC;AAE3G,SAAO;AAAA,IACL,OAAO,KAAK,OAAO,QAAQ,SAAS,OAAO;AAAA,IAC3C,IAAI,KAAK,OAAO,KAAK,SAAS,OAAO;AAAA,IACrC,WAAW,MAAM,SAAS,OAAO,WAAW,KAAK,OAAO,SAAS;AAAA,IACjE,QAAQ,MAAM,SAAS,OAAO,QAAQ,KAAK,OAAO,MAAM;AAAA,IACxD,cAAc,KAAK,OAAO,MAAM,eAAe,SAAS,OAAO,MAAM;AAAA,IACrE,kBAAkB,KAAK,OAAO,MAAM,mBAAmB,SAAS,OAAO,MAAM;AAAA,IAC7E,GAAI,KAAK,OAAO,SAAS,UAAa,SAAS,OAAO,SAAS,SAC3D,EAAE,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO,KAAK,IAChD,CAAC;AAAA,IACL,OAAO,KAAK,OAAO,UAAU,MAAM,SAAS,OAAO,UAAU;AAAA,IAC7D;AAAA,EACF;AACF;AAKA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,eAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAC3E;AAEA,SAAS,OAAO,OAAsB,SAAS,GAAG,SAAS,IAAY;AACrE,MAAI,UAAU,QAAQ,UAAU,EAAG,QAAO;AAC1C,QAAM,OAAO,SAAS,IAAI,MAAM,QAAQ,MAAM,IAAI,OAAO,KAAK;AAC9D,SAAO,KAAK,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,GAAG,MAAM;AAClD;AAEA,SAAS,UAAU,OAA8B;AAC/C,SAAO,UAAU,QAAQ,UAAU,IAAI,KAAK,KAAK,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAClG;AAGO,SAAS,aAAa,QAAoB,MAAyB;AACxE,QAAM,IAAI,OAAO;AACjB,QAAM,QAAkB,CAAC;AACzB,QAAM;AAAA,IACJ,SAAS,OAAO,QAAQ,KAAK,OAAO,IAAI,YAAO,EAAE,QAAQ,gBAAgB,EAAE,EAAE,OAAO,OAAO,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,SAAS,OAAO,MAAM,SAAS,IAAI,CAAC;AAAA,EAC9J;AACA,QAAM,KAAK,EAAE;AAEb,QAAM,QAAQ,KAAK,IAAI,GAAG,GAAG,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC;AACpE,QAAM,SAAS,IAAI,KAAK,MAAM,UAAU,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AACnE,QAAM,KAAK,GAAG,QAAQ,OAAO,KAAK,CAAC,qCAAgC;AACnE,QAAM,MAAM,CAAC,OAAsB,MAAc,IAAI,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC;AAC7E,aAAW,SAAS,OAAO,QAAQ;AACjC,UAAM,IAAI,OAAO,IAAI,MAAM,IAAI;AAC/B,UAAM,SAAS,IACX,CAAC,EAAE,cAAc,QAAQ,EAAE,cAAc,IAAI,IAAI,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,WAAW,QAAQ,EAAE,WAAW,IAAI,IAAI,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,EAC/J,OAAO,OAAO,EACd,KAAK,GAAG,IACX;AACJ,UAAM;AAAA,MACJ,GAAG,MAAM,KAAK,OAAO,KAAK,CAAC,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC,KAAK,IAAI,MAAM,QAAQ,CAAC,CAAC,MAAM,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,OAAO,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,KAAK,MAAM;AAAA,IACjM;AAAA,EACF;AACA,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,sBAAsB,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,aAAa,IAAI,CAAC,YAAY,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,UAAU,MAAM,UAAU,IAAI,CAAC;AAAA,EACtJ;AACA,QAAM,IAAI,EAAE;AACZ,QAAM,QAAQ,EAAE,mBAAmB,EAAE,mBACjC,gBAAgB,EAAE,gBAAgB,eAAe,OAAO,CAAC,YAAY,EAAE,iBAAiB,eAAe,OAAO,CAAC,KAC/G;AACJ,QAAM;AAAA,IACJ,WAAW,EAAE,aAAa,eAAe,OAAO,CAAC,UAAU,OAAO,MAAM,gBAAgB,IAAI,CAAC,MAAM,EAAE,iBAAiB,eAAe,OAAO,CAAC,cAAc,OAAO,MAAM,oBAAoB,IAAI,CAAC,GAAG,KAAK,KAAK,EAAE,KAAK,cAClN,EAAE,SAAS,SAAY,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAC,GAAG,OAAO,MAAM,QAAQ,MAAM,CAAC,CAAC,KAAK;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,gBAAgB,EAAE,UAAU,GAAG,KAAK,OAAO,MAAM,SAAS,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,EAAE,UAAU,GAAG;AAAA,EAC5H;AAEA,QAAM,SAAS,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE;AAC/C,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,SAAS;AACpB,eAAW,QAAQ,OAAQ,OAAM,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,EAAE;AAAA,EACvE;AACA,QAAM,YAAY,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK;AAC7D,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,aAAa;AACxB,eAAW,QAAQ,WAAW;AAC5B,iBAAW,QAAQ,KAAK,OAAO,OAAO,CAAC,MAAM,EAAE,YAAY,OAAO,GAAG;AACnE,cAAM,SACJ,KAAK,YAAY,UACb,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,SAAS,KAAK,UAAU,KAAK,MAAM,CAAC,KAC7E,KAAK,YAAY,YACf,YAAY,KAAK,UAAU,KAAK,QAAQ,CAAC,kBACzC,cAAc,KAAK,UAAU,KAAK,MAAM,CAAC;AACjD,cAAM,OAAO,KAAK,aAAa,KAAK,KAAK,UAAU,MAAM;AACzD,cAAM,KAAK,KAAK,KAAK,IAAI,WAAM,KAAK,IAAI,KAAK,MAAM,GAAG,IAAI,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":["readdirSync","readFileSync","existsSync","mkdirSync","writeFileSync","join","stable"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sembl/testing",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Record/replay providers and an eval harness for SEMBL: deterministic tests and measurable prompt changes.",
   "keywords": [
     "llm",
@@ -53,7 +53,7 @@
     "provenance": true
   },
   "dependencies": {
-    "@sembl/core": "0.2.0"
+    "@sembl/core": "0.3.0"
   },
   "devDependencies": {
     "@types/node": "^25.5.0",