@struktur/benchmarks 2.6.0

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/types.ts","../src/config.ts","../src/pricing.ts","../src/tracks.ts","../src/scoring/normalize.ts","../src/scoring/score.ts","../src/runner.ts","../src/cache.ts","../src/report.ts"],"sourcesContent":["import type { Artifact, StandardSchema, TypedJSONSchema, Usage } from \"@struktur/sdk\";\n\n/**\n * The four ways a source can be presented to the model.\n *\n * Text is always present — the PDF parser extracts text unconditionally. The\n * track only varies which images accompany it:\n * - `text`                       → text only\n * - `text+embedded`              → text + images embedded in the source\n * - `text+screenshots`           → text + rendered page screenshots\n * - `text+embedded+screenshots`  → text + both\n *\n * For image-only sources (scanned receipts, photos) text is absent and the\n * image IS the content, so only `text+embedded` is meaningful.\n */\nexport type Track = \"text\" | \"text+embedded\" | \"text+screenshots\" | \"text+embedded+screenshots\";\n\nexport const TRACKS: readonly Track[] = [\n  \"text\",\n  \"text+embedded\",\n  \"text+screenshots\",\n  \"text+embedded+screenshots\",\n] as const;\n\n/**\n * Infer the output type `T` of a schema:\n * - Standard Schema v1 (Zod, Valibot, ArkType, …) → its output type\n * - TypedJSONSchema<T> (tagged plain JSON Schema)      → T\n * - plain JSON Schema object                            → unknown (no type info carried)\n */\nexport type SchemaOutput<S> =\n  S extends StandardSchema<unknown, infer T> ? T : S extends TypedJSONSchema<infer T> ? T : unknown;\n\n/** How a single field is compared against its gold value. */\nexport type FieldMetric = \"exact\" | \"normalized\" | \"tolerance\" | \"semantic\" | \"prose\";\n\n/** An array path (dot notation) aligned by per-item key field(s) instead of index. */\nexport type ArrayAlignment = {\n  /** e.g. `\"items\"` or `\"invoice.items\"` */\n  path: string;\n  /** Field(s) within each item used to match items, e.g. `\"name\"` or `[\"floor\",\"area\"]`. */\n  key: string | string[];\n};\n\n/**\n * Per-field scoring rules. Defaults to `normalized` (case/whitespace/diacritic\n * folding, numeric equality) when nothing is specified.\n */\nexport type FieldMetricSpec = {\n  default?: FieldMetric;\n  /** Dot-path → metric override. e.g. `{ \"total\": \"tolerance\" }` */\n  fields?: Record<string, FieldMetric>;\n  /** Arrays matched by a key field rather than position. */\n  arrays?: ArrayAlignment[];\n  /** Array paths scored as order-insensitive sets (uses the def value metric). */\n  sets?: string[];\n};\n\n/** Provenance metadata attached to a case (license, attribution). */\nexport type CaseSourceInfo = {\n  dataset?: string;\n  license?: string;\n  url?: string;\n  attribution?: string;\n};\n\n/**\n * The portable unit of the benchmark: one input → one schema → one correct\n * answer. `gold` is a fully-populated instance of `schema` — the exact JSON a\n * perfect extraction should return.\n *\n * `artifacts` is the base input. For PDF sources the runner re-parses\n * `artifacts` per track (see `materializeTrack`); for image/text sources that\n * can't be re-derived, supply `artifactsByTrack` to pin specific inputs.\n */\nexport interface BenchmarkCase<TSchema = unknown, T = SchemaOutput<TSchema>> {\n  id: string;\n  schema: TSchema;\n  gold: T;\n  artifacts: Artifact[];\n  /** Tracks this case can produce. Defaults to all four. */\n  tracks?: Track[];\n  /**\n   * Pre-materialized artifacts per track. Takes precedence over `artifacts`\n   * (which is still required as the fallback/default). Use when a source can't\n   * be re-parsed from its raw bytes, e.g. image-only datasets.\n   */\n  artifactsByTrack?: Partial<Record<Track, Artifact[]>>;\n  /** Per-field scoring rules for this case. */\n  metrics?: FieldMetricSpec;\n  /**\n   * Optional normalizer applied to a prediction before scoring (and before\n   * caching). Use to make gold/prediction shapes symmetric, e.g. collapsing\n   * building+unit features into one set. Gold is NOT passed through this.\n   */\n  transform?: (data: unknown) => unknown;\n  source?: CaseSourceInfo;\n}\n\n/** Identity helper that preserves schema → gold type inference. */\nexport function defineCase<TSchema>(c: BenchmarkCase<TSchema>): BenchmarkCase<TSchema> {\n  return c;\n}\n\nexport type { Usage };\n","/**\n * The benchmark measures the harness, not the model — the model is a fixed baseline.\n *\n * deepseek-v4-flash-vision-exp: ~$0.60/M tokens, vision-capable, supports\n * structured output (json_schema) via OpenRouter. Cheap enough for frequent\n * benchmark runs while testing the full matrix of text + image variants.\n */\nexport const BENCHMARK_MODEL = \"openrouter/deepseek/deepseek-v4-flash-vision-exp\";\n\n/** Default cache root for downloaded datasets. */\nexport const defaultDatasetDir = (): string =>\n  process.env.STRUKTUR_BENCHMARKS_DIR ?? `${process.env.HOME ?? \".\"}/.struktur/benchmarks`;\n\n/** Default cache root for extraction result cells. */\nexport const defaultResultsDir = (): string => `${defaultDatasetDir()}/results`;\n","import type { Usage } from \"@struktur/sdk\";\n\n/**\n * Token pricing in USD per 1M tokens, keyed by the benchmark model string\n * (`provider/model`). Prices are snapshot baselines; the harness, not the\n * model, is what's measured, so a small table covering the pinned models is\n * enough. Override with STRUKTUR_BENCHMARK_PRICES (JSON: `{ \"<model>\": { input, output } }`).\n */\n\nexport type ModelPricing = { input: number; output: number };\n\nconst PRICING: Record<string, ModelPricing> = {\n  // OpenRouter: $0.22/M input, $0.66/M output\n  \"openrouter/deepseek/deepseek-v4-flash-vision-exp\": { input: 0.22, output: 0.66 },\n  // OpenAI via OpenRouter: $0.15/M input, $0.60/M output\n  \"openrouter/openai/gpt-4o-mini\": { input: 0.15, output: 0.6 },\n};\n\nconst envOverride = ((): Record<string, ModelPricing> | undefined => {\n  const raw = process.env.STRUKTUR_BENCHMARK_PRICES;\n  if (!raw) return undefined;\n  try {\n    return JSON.parse(raw) as Record<string, ModelPricing>;\n  } catch {\n    return undefined;\n  }\n})();\n\n/** Strip an OpenRouter `#provider` suffix (`model#provider` → `model`). */\nconst normalizeModel = (model: string): string => model.replace(/#.*$/, \"\");\n\nexport function pricingFor(model: string): ModelPricing | undefined {\n  const key = normalizeModel(model);\n  return envOverride?.[key] ?? PRICING[key];\n}\n\n/** Estimated USD cost of a token usage for a model. 0 when pricing is unknown. */\nexport function estimateCostUsd(model: string, usage: Usage): number {\n  const p = pricingFor(model);\n  if (!p) return 0;\n  return (usage.inputTokens / 1_000_000) * p.input + (usage.outputTokens / 1_000_000) * p.output;\n}\n","import { parsePdf } from \"@struktur/sdk\";\nimport type { Artifact } from \"@struktur/sdk\";\nimport type { Track } from \"./types\";\n\n/** Map a track to the PDF parse flags that produce it. */\nexport const trackToPdfOptions = (\n  track: Track,\n): { includeImages: boolean; screenshots: boolean } => {\n  switch (track) {\n    case \"text\":\n      return { includeImages: false, screenshots: false };\n    case \"text+embedded\":\n      return { includeImages: true, screenshots: false };\n    case \"text+screenshots\":\n      return { includeImages: false, screenshots: true };\n    case \"text+embedded+screenshots\":\n      return { includeImages: true, screenshots: true };\n  }\n};\n\n/**\n * Produce the artifact set for a track.\n *\n * PDF artifacts are re-parsed from their raw bytes with the track's image\n * flags (the parser always keeps text). Image/text/file artifacts are returned\n * unchanged — their media is intrinsic and can't be toggled off.\n */\nexport async function materializeTrack(artifacts: Artifact[], track: Track): Promise<Artifact[]> {\n  return Promise.all(\n    artifacts.map(async (artifact) => {\n      if (artifact.type !== \"pdf\") return artifact;\n      const buffer = await artifact.raw();\n      return parsePdf(buffer, trackToPdfOptions(track));\n    }),\n  );\n}\n","import type { FieldMetric } from \"../types\";\n\n/** Sentinel for \"no value\": null, undefined, and absent keys all normalize here. */\nexport const EMPTY = Symbol(\"struktur.benchmarks.empty\");\n\nexport type Normalized = string | number | boolean | typeof EMPTY;\n\nconst stripDiacritics = (s: string): string => s.normalize(\"NFD\").replace(/[\\u0300-\\u036f]/g, \"\");\n\n/** Case, whitespace, and diacritic folding for normal string comparison. */\nexport const normalizeString = (s: string): string =>\n  stripDiacritics(s.toLowerCase()).replace(/\\s+/g, \" \").trim();\n\n/** Aggressive folding for semantic equivalence: alphanumerics only. */\nexport const normalizeAggressive = (s: string): string =>\n  stripDiacritics(s.toLowerCase()).replace(/[^a-z0-9]+/g, \"\");\n\n/**\n * Normalize a leaf value according to a metric. Leaves are primitives\n * (string/number/boolean) or null/undefined; anything else is stringified so a\n * type mismatch still produces a deterministic, non-equal comparison.\n */\nexport function normalizeValue(value: unknown, metric: FieldMetric): Normalized {\n  if (value === null || value === undefined) return EMPTY;\n\n  if (metric === \"exact\") {\n    if (typeof value === \"string\" || typeof value === \"number\" || typeof value === \"boolean\") {\n      return value;\n    }\n    return JSON.stringify(value);\n  }\n\n  if (metric === \"semantic\") {\n    if (typeof value === \"number\" || typeof value === \"boolean\") return value;\n    return normalizeAggressive(typeof value === \"string\" ? value : JSON.stringify(value));\n  }\n\n  // normalized / tolerance\n  if (typeof value === \"number\" || typeof value === \"boolean\") return value;\n  return normalizeString(typeof value === \"string\" ? value : JSON.stringify(value));\n}\n\nconst RELATIVE_TOLERANCE = 1e-6;\n\n/**\n * Compare two leaf values under a metric. `EMPTY == EMPTY` is always true:\n * extracting \"nothing\" where the source has nothing is correct.\n */\nexport function valuesEqual(a: unknown, b: unknown, metric: FieldMetric): boolean {\n  const na = normalizeValue(a, metric);\n  const nb = normalizeValue(b, metric);\n\n  if (na === EMPTY || nb === EMPTY) return na === EMPTY && nb === EMPTY;\n\n  if (metric === \"tolerance\" && typeof na === \"number\" && typeof nb === \"number\") {\n    const scale = Math.max(1, Math.abs(na), Math.abs(nb));\n    return Math.abs(na - nb) <= RELATIVE_TOLERANCE * scale;\n  }\n\n  return na === nb;\n}\n","import type { ArrayAlignment, FieldMetric, FieldMetricSpec } from \"../types\";\nimport { EMPTY, normalizeValue, valuesEqual } from \"./normalize\";\n\n// ---- Prose coverage scoring -------------------------------------------------\n// description_text / location_text are free prose. Comparing them verbatim is\n// meaningless (two writers never produce identical prose). Score them by\n// content-word overlap — an ordered F1 over the distinct content tokens.\n\nconst STOPWORDS = new Set([\n  \"the\",\n  \"a\",\n  \"an\",\n  \"and\",\n  \"or\",\n  \"of\",\n  \"to\",\n  \"in\",\n  \"on\",\n  \"at\",\n  \"is\",\n  \"are\",\n  \"it\",\n  \"this\",\n  \"that\",\n  \"with\",\n  \"for\",\n  \"as\",\n  \"by\",\n  \"from\",\n  \"be\",\n  \"was\",\n  \"were\",\n  \"und\",\n  \"der\",\n  \"die\",\n  \"das\",\n  \"ein\",\n  \"eine\",\n  \"einen\",\n  \"dem\",\n  \"den\",\n  \"mit\",\n  \"von\",\n  \"zur\",\n  \"zum\",\n  \"auf\",\n  \"im\",\n  \"in\",\n  \"am\",\n  \"ist\",\n  \"sind\",\n  \"werden\",\n  \"wird\",\n  \"des\",\n  \"sich\",\n  \"auch\",\n  \"nicht\",\n  \"als\",\n  \"bei\",\n  \"für\",\n  \"über\",\n  \"aus\",\n  \"nach\",\n  \"einer\",\n  \"einem\",\n  \"eine\",\n  \"zu\",\n  \"unter\",\n  \"an\",\n  \"sowie\",\n  \"durch\",\n  \"alle\",\n  \"dass\",\n  \"dieser\",\n  \"diese\",\n  \"hier\",\n  \"sein\",\n  \"ihr\",\n  \"wir\",\n  \"man\",\n]);\n\nconst tokenizeProse = (s: string): string[] => {\n  if (typeof s !== \"string\") return [];\n  return Array.from(\n    new Set(\n      s\n        .toLowerCase()\n        .split(/[^\\p{L}\\p{N}]+/u)\n        .filter((w) => w.length >= 3 && !STOPWORDS.has(w)),\n    ),\n  );\n};\n\nfunction scoreProse(gold: unknown, pred: unknown): FieldScore {\n  const s = empty();\n  const gt = tokenizeProse(String(gold ?? \"\"));\n  const pt = tokenizeProse(String(pred ?? \"\"));\n  const gs = new Set(gt);\n  const ps = new Set(pt);\n  const tp = gt.filter((w) => ps.has(w)).length;\n  const fp = pt.filter((w) => !gs.has(w)).length;\n  const fn = gt.filter((w) => !ps.has(w)).length;\n  s.tp = tp;\n  s.fp = fp;\n  s.fn = fn;\n  s.totalGold = gt.length;\n  s.totalPred = pt.length;\n  return finalize(s);\n}\n\n/**\n * Field-level score. Fields (leaves) are compared individually so omission\n * (gold field missing) and hallucination (extra predicted field) are counted\n * separately, and a wrong value counts against both precision and recall.\n */\nexport type FieldScore = {\n  /** Correctly extracted leaf values. */\n  tp: number;\n  /** Predicted leaves that were wrong (hallucinated or wrong value). */\n  fp: number;\n  /** Gold leaves that were wrong (omitted or wrong value). */\n  fn: number;\n  totalGold: number;\n  totalPred: number;\n  precision: number;\n  recall: number;\n  f1: number;\n  /** True when every gold field matched exactly and no field was hallucinated. */\n  exactMatch: boolean;\n  /** Per-field mismatches for debugging. */\n  fieldErrors: FieldError[];\n};\n\nexport type FieldError = {\n  path: string;\n  gold: unknown;\n  pred: unknown;\n  metric: FieldMetric;\n};\n\nconst empty = (): FieldScore => ({\n  tp: 0,\n  fp: 0,\n  fn: 0,\n  totalGold: 0,\n  totalPred: 0,\n  precision: 1,\n  recall: 1,\n  f1: 1,\n  exactMatch: true,\n  fieldErrors: [],\n});\n\nconst finalize = (s: FieldScore): FieldScore => {\n  s.precision = s.tp + s.fp === 0 ? 1 : s.tp / (s.tp + s.fp);\n  s.recall = s.tp + s.fn === 0 ? 1 : s.tp / (s.tp + s.fn);\n  s.f1 = s.precision + s.recall === 0 ? 0 : (2 * s.precision * s.recall) / (s.precision + s.recall);\n  s.exactMatch = s.fp === 0 && s.fn === 0;\n  return s;\n};\n\nconst merge = (into: FieldScore, child: FieldScore): void => {\n  into.tp += child.tp;\n  into.fp += child.fp;\n  into.fn += child.fn;\n  into.totalGold += child.totalGold;\n  into.totalPred += child.totalPred;\n  into.fieldErrors.push(...child.fieldErrors);\n};\n\nconst isLeaf = (v: unknown): boolean =>\n  v === null ||\n  v === undefined ||\n  typeof v === \"string\" ||\n  typeof v === \"number\" ||\n  typeof v === \"boolean\";\n\nconst isObject = (v: unknown): v is Record<string, unknown> =>\n  typeof v === \"object\" && v !== null && !Array.isArray(v);\n\n/** Number of leaf values a (sub)tree contains. `{}` has zero; `null` counts as one. */\nexport const leafCount = (v: unknown): number => {\n  if (Array.isArray(v)) return v.reduce<number>((n, item) => n + leafCount(item), 0);\n  if (isObject(v)) {\n    return Object.values(v).reduce<number>((n, item) => n + leafCount(item), 0);\n  }\n  return 1;\n};\n\n/** Collapse concrete array keys to `[]` so nested alignment/set lookups work. */\nconst canonPath = (path: string): string => path.replace(/\\[[^\\]]*\\]/g, \"[]\");\n\nconst metricFor = (spec: FieldMetricSpec, path: string): FieldMetric =>\n  // field overrides are keyed by the leaf field name (e.g. \"area\"), robust to\n  // concrete array keys in the path.\n  spec.fields?.[path.split(\".\").pop() as string] ?? spec.default ?? \"normalized\";\n\nconst alignmentFor = (spec: FieldMetricSpec, path: string): ArrayAlignment | undefined =>\n  spec.arrays?.find((a) => canonPath(a.path) === canonPath(path)) ??\n  spec.arrays?.find((a) => canonPath(path).endsWith(canonPath(a.path)));\n\nconst isSetPath = (spec: FieldMetricSpec, path: string): boolean =>\n  (spec.sets ?? []).some(\n    (s) => canonPath(s) === canonPath(path) || canonPath(path).endsWith(canonPath(s)),\n  );\n\nfunction groupByKey(items: unknown[], key: string | string[]): Map<string, unknown[]> {\n  const map = new Map<string, unknown[]>();\n  const keys = Array.isArray(key) ? key : [key];\n  for (const item of items) {\n    const k = keys\n      .map((kf) => (isObject(item) ? item[kf] : undefined))\n      .map((v) => String(normalizeValue(v, \"normalized\")))\n      .join(\"|\");\n    const arr = map.get(k) ?? [];\n    arr.push(item);\n    map.set(k, arr);\n  }\n  return map;\n}\n\n/** Compare a leaf pair (at least one side is a leaf/null/undefined). */\nfunction scoreLeaf(path: string, gold: unknown, pred: unknown, spec: FieldMetricSpec): FieldScore {\n  const metric = metricFor(spec, path);\n  if (metric === \"prose\" && (typeof gold === \"string\" || typeof pred === \"string\")) {\n    return scoreProse(gold, pred);\n  }\n  const s = empty();\n  s.totalGold = isLeaf(gold) ? 1 : leafCount(gold);\n  s.totalPred = isLeaf(pred) ? 1 : leafCount(pred);\n\n  if (valuesEqual(gold, pred, metric)) {\n    s.tp = isLeaf(gold) ? 1 : leafCount(gold);\n    // both sides equal → the gold side is fully correct; nothing wrong.\n    return finalize(s);\n  }\n\n  const goldEmpty = gold === null || gold === undefined;\n  const predEmpty = pred === null || pred === undefined;\n  if (goldEmpty) s.fp = s.totalPred;\n  else if (predEmpty) s.fn = s.totalGold;\n  else {\n    s.fn = s.totalGold;\n    s.fp = s.totalPred;\n  }\n  s.fieldErrors.push({ path, gold, pred, metric });\n  return finalize(s);\n}\n\nfunction scoreIndexed(path: string, g: unknown[], p: unknown[], spec: FieldMetricSpec): FieldScore {\n  const s = empty();\n  const len = Math.max(g.length, p.length);\n  for (let i = 0; i < len; i++) {\n    merge(s, scoreAt(`${path}[${i}]`, g[i] ?? null, p[i] ?? null, spec));\n  }\n  return finalize(s);\n}\n\nfunction scoreAligned(\n  path: string,\n  g: unknown[],\n  p: unknown[],\n  key: string | string[],\n  spec: FieldMetricSpec,\n): FieldScore {\n  const s = empty();\n  const gm = groupByKey(g, key);\n  const pm = groupByKey(p, key);\n\n  // If no key overlaps at all (e.g. building label \"Halle 1\" vs\n  // \"Halle 1 mit Bürogebäude\"), key alignment scores everything as FN+FP.\n  // Fall back to positional alignment — item order follows the document in\n  // both gold and prediction, which is a better match than nothing.\n  let matchedAny = false;\n\n  for (const [k, gItems] of gm) {\n    const pItems = pm.get(k);\n    if (!pItems || pItems.length === 0) {\n      for (const gi of gItems) {\n        s.fn += leafCount(gi);\n        s.totalGold += leafCount(gi);\n      }\n      continue;\n    }\n    matchedAny = true;\n    const pairs = Math.min(gItems.length, pItems.length);\n    for (let i = 0; i < pairs; i++) {\n      merge(s, scoreAt(`${path}[${String(k)}]`, gItems[i], pItems[i], spec));\n    }\n    for (let i = pairs; i < gItems.length; i++) {\n      s.fn += leafCount(gItems[i]!);\n      s.totalGold += leafCount(gItems[i]!);\n    }\n    for (let i = pairs; i < pItems.length; i++) {\n      s.fp += leafCount(pItems[i]!);\n      s.totalPred += leafCount(pItems[i]!);\n    }\n  }\n\n  for (const [k, pItems] of pm) {\n    if (gm.has(k)) continue;\n    for (const pi of pItems) {\n      s.fp += leafCount(pi);\n      s.totalPred += leafCount(pi);\n    }\n  }\n\n  if (!matchedAny && g.length > 0 && p.length > 0) {\n    return scoreIndexed(path, g, p, spec);\n  }\n\n  return finalize(s);\n}\n\nfunction scoreSet(path: string, g: unknown[], p: unknown[], spec: FieldMetricSpec): FieldScore {\n  // Compare as an order-insensitive multiset. Duplicates count independently.\n  const s = empty();\n  const goldKeyed = new Map<string, number>();\n  const predKeyed = new Map<string, number>();\n  const metric = metricFor(spec, canonPath(path));\n  const bump = (map: Map<string, number>, v: unknown) => {\n    const k = String(normalizeValue(v, metric));\n    map.set(k, (map.get(k) ?? 0) + 1);\n  };\n\n  for (const v of g) bump(goldKeyed, v);\n  for (const v of p) bump(predKeyed, v);\n\n  s.totalGold = g.length;\n  s.totalPred = p.length;\n\n  for (const [k, gc] of goldKeyed) {\n    const pc = predKeyed.get(k) ?? 0;\n    const common = Math.min(gc, pc);\n    s.tp += common;\n    s.fn += gc - common;\n    s.fp += pc - common;\n    if (common < gc) {\n      s.fieldErrors.push({\n        path,\n        gold: g.find((v) => String(normalizeValue(v, metric)) === k),\n        pred: null,\n        metric,\n      });\n    }\n  }\n  for (const [k, pc] of predKeyed) {\n    const gc = goldKeyed.get(k) ?? 0;\n    if (pc > gc) {\n      s.fieldErrors.push({\n        path,\n        gold: null,\n        pred: p.find((v) => String(normalizeValue(v, metric)) === k),\n        metric,\n      });\n    }\n  }\n  return finalize(s);\n}\n\nfunction scoreAt(path: string, gold: unknown, pred: unknown, spec: FieldMetricSpec): FieldScore {\n  if (isLeaf(gold) || isLeaf(pred)) {\n    return scoreLeaf(path, gold, pred, spec);\n  }\n\n  if (Array.isArray(gold) || Array.isArray(pred)) {\n    const g = Array.isArray(gold) ? gold : [gold];\n    const p = Array.isArray(pred) ? pred : [pred];\n    if (isSetPath(spec, path)) {\n      return scoreSet(path, g, p, spec);\n    }\n    const align = alignmentFor(spec, path);\n    return align ? scoreAligned(path, g, p, align.key, spec) : scoreIndexed(path, g, p, spec);\n  }\n\n  const s = empty();\n  const keys = new Set([...Object.keys(gold as object), ...Object.keys(pred as object)]);\n  for (const key of keys) {\n    const childPath = path === \"\" ? key : `${path}.${key}`;\n    const gv = (gold as Record<string, unknown>)[key] ?? null;\n    const pv = (pred as Record<string, unknown>)[key] ?? null;\n    merge(s, scoreAt(childPath, gv, pv, spec));\n  }\n  return finalize(s);\n}\n\n/**\n * Score an extraction output against its gold instance, field by field.\n */\nexport function scoreData(gold: unknown, pred: unknown, spec: FieldMetricSpec = {}): FieldScore {\n  return scoreAt(\"\", gold, pred, spec);\n}\n\n/** Export the sentinel so tests/consumers can build explicit EMPTY comparisons. */\nexport { EMPTY };\n","import type { ExtractionStrategy, Usage } from \"@struktur/sdk\";\nimport { extract, resolveModel, toJsonSchema } from \"@struktur/sdk\";\nimport {\n  simple,\n  parallel,\n  sequential,\n  parallelAutoMerge,\n  sequentialAutoMerge,\n  doublePass,\n  doublePassAutoMerge,\n  agent,\n} from \"@struktur/sdk\";\nimport type { BenchmarkCase, Track } from \"./types\";\nimport { TRACKS } from \"./types\";\nimport { materializeTrack } from \"./tracks\";\nimport { scoreData, type FieldScore } from \"./scoring/score\";\nimport { cacheGet, cacheSet, hashKey } from \"./cache\";\nimport { BENCHMARK_MODEL, defaultResultsDir } from \"./config\";\nimport { estimateCostUsd } from \"./pricing\";\n\n// ---------------------------------------------------------------------------\n// Strategy resolution\n// ---------------------------------------------------------------------------\n\ntype BuiltinName =\n  | \"simple\"\n  | \"parallel\"\n  | \"sequential\"\n  | \"parallelAutoMerge\"\n  | \"sequentialAutoMerge\"\n  | \"doublePass\"\n  | \"doublePassAutoMerge\"\n  | \"agent\";\n\n/**\n * Strategy factory: the runner resolves the model once and injects it.\n * `instructions` maps to `outputInstructions` — accept it if you want prompt\n * variants, otherwise ignore it.\n */\nexport type StrategyFactory<T = unknown> = (\n  model: unknown,\n  modelSpec: string,\n  instructions?: string,\n) => ExtractionStrategy<T>;\n\n/** A built-in strategy plus an optional prompt variant and report label. */\nexport type StrategyEntry = {\n  label?: string;\n  instructions?: string;\n  strategy: BuiltinName | StrategyFactory;\n};\n\nexport type StrategyLike =\n  | BuiltinName\n  | StrategyFactory\n  | ExtractionStrategy<unknown>\n  | StrategyEntry;\n\n// ---------------------------------------------------------------------------\n// Built-in factories\n// ---------------------------------------------------------------------------\n\nconst builtins: Record<BuiltinName, StrategyFactory> = {\n  simple: (model, _spec, instructions) => simple({ model, outputInstructions: instructions }),\n  parallel: (model, _spec, instructions) =>\n    parallel({ model, mergeModel: model, chunkSize: 10_000, outputInstructions: instructions }),\n  sequential: (model, _spec, instructions) =>\n    sequential({ model, chunkSize: 10_000, outputInstructions: instructions }),\n  parallelAutoMerge: (model, _spec, instructions) =>\n    parallelAutoMerge({\n      model,\n      dedupeModel: model,\n      chunkSize: 10_000,\n      outputInstructions: instructions,\n    }),\n  sequentialAutoMerge: (model, _spec, instructions) =>\n    sequentialAutoMerge({\n      model,\n      dedupeModel: model,\n      chunkSize: 10_000,\n      outputInstructions: instructions,\n    }),\n  doublePass: (model, _spec, instructions) =>\n    doublePass({ model, mergeModel: model, chunkSize: 10_000, outputInstructions: instructions }),\n  doublePassAutoMerge: (model, _spec, instructions) =>\n    doublePassAutoMerge({\n      model,\n      dedupeModel: model,\n      chunkSize: 10_000,\n      outputInstructions: instructions,\n    }),\n  agent: (_model, modelSpec, instructions) => {\n    const [provider, ...rest] = modelSpec.split(\"/\");\n    const modelId = rest.join(\"/\");\n    if (!provider || !modelId)\n      throw new Error(`Agent requires 'provider/model'. Got: ${modelSpec}`);\n    return agent({\n      provider,\n      modelId,\n      maxSteps: 50,\n      maxIterations: 1,\n      vision: true,\n      outputInstructions: instructions,\n    });\n  },\n};\n\n// ---------------------------------------------------------------------------\n// Runner types\n// ---------------------------------------------------------------------------\n\nexport type RunOptions = {\n  cases: BenchmarkCase[];\n  strategies: StrategyLike[];\n  /** Model string in `provider/model` form. Defaults to the pinned benchmark model. */\n  model?: string;\n  /** Cache directory for result cells. Defaults to `~/.struktur/benchmarks/results`. */\n  cacheDir?: string;\n  /** Set to `false` to skip the disk cache. */\n  cache?: boolean;\n  /** Opaque variant tag included in the cache key — bump to invalidate. */\n  variant?: string;\n  onCell?: (info: { caseId: string; track: Track; strategy: string; cached: boolean }) => void;\n};\n\nexport type CellResult = {\n  caseId: string;\n  track: Track;\n  strategy: string;\n  model: string;\n  cached: boolean;\n  valid: boolean;\n  error?: string;\n  score: FieldScore;\n  usage: Usage;\n  latencyMs: number;\n  /** Estimated USD cost of the cell's token usage (0 if pricing unknown). */\n  costUsd?: number;\n  /** Raw extracted data (null if extraction failed). */\n  data: unknown;\n};\n\nexport type SummaryRow = {\n  strategy: string;\n  track: Track;\n  cases: number;\n  meanF1: number;\n  meanPrecision: number;\n  meanRecall: number;\n  validityRate: number;\n  exactMatchRate: number;\n  totalInputTokens: number;\n  totalOutputTokens: number;\n  /** Sum of cell costs across the row (USD). */\n  totalCostUsd: number;\n  /** Mean cost per case (USD). */\n  meanCostUsd: number;\n  /** Total wall-clock latency across all cells in the row (ms). */\n  totalLatencyMs: number;\n  meanLatencyMs: number;\n};\n\nexport type BenchmarkReport = {\n  model: string;\n  variant?: string;\n  generatedAt: string;\n  cells: CellResult[];\n  summary: SummaryRow[];\n};\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\nfunction resolveStrategy(\n  entry: StrategyLike,\n  model: unknown,\n  modelSpec: string,\n): { name: string; strategy: ExtractionStrategy<unknown> } {\n  if (typeof entry === \"string\") {\n    const factory = builtins[entry];\n    if (!factory) {\n      throw new Error(\n        `Unknown builtin strategy: ${entry}. Available: ${Object.keys(builtins).join(\", \")}`,\n      );\n    }\n    return { name: entry, strategy: factory(model, modelSpec) };\n  }\n\n  if (typeof entry === \"function\") {\n    const s = entry(model, modelSpec);\n    return { name: s.name, strategy: s };\n  }\n\n  if (\"strategy\" in entry) {\n    const inner = entry.strategy;\n    if (typeof inner === \"string\") {\n      const factory = builtins[inner];\n      if (!factory) throw new Error(`Unknown builtin strategy: ${inner}`);\n      return {\n        name: entry.label ?? inner,\n        strategy: factory(model, modelSpec, entry.instructions),\n      };\n    }\n    const s = inner(model, modelSpec, entry.instructions);\n    return { name: entry.label ?? s.name, strategy: s };\n  }\n\n  return {\n    name: (entry as ExtractionStrategy<unknown>).name,\n    strategy: entry as ExtractionStrategy<unknown>,\n  };\n}\n\nconst emptyUsage = (): Usage => ({ inputTokens: 0, outputTokens: 0, totalTokens: 0 });\n\n// ---------------------------------------------------------------------------\n// Main\n// ---------------------------------------------------------------------------\n\nexport async function runBenchmark(options: RunOptions): Promise<BenchmarkReport> {\n  const modelSpec = options.model ?? BENCHMARK_MODEL;\n  const model = await resolveModel(modelSpec);\n  const cacheDir = options.cacheDir ?? defaultResultsDir();\n  const useCache = options.cache ?? true;\n  const variant = options.variant ?? \"\";\n\n  const strats = options.strategies.map((s) => resolveStrategy(s, model, modelSpec));\n  const cells: CellResult[] = [];\n\n  for (const c of options.cases) {\n    const tracks = c.tracks ?? [...TRACKS];\n    for (const track of tracks) {\n      for (const { name, strategy } of strats) {\n        const cacheKey = hashKey({\n          caseId: c.id,\n          track,\n          strategy: name,\n          model: modelSpec,\n          variant,\n          schema: toJsonSchema(c.schema),\n          gold: c.gold,\n        });\n\n        const cached = useCache ? await cacheGet<CellResult>(cacheDir, cacheKey) : undefined;\n        if (cached) {\n          cached.cached = true;\n          // Backfill cost for cells cached before cost tracking was added.\n          if (cached.costUsd === undefined)\n            cached.costUsd = estimateCostUsd(modelSpec, cached.usage);\n          cells.push(cached);\n          options.onCell?.({ caseId: c.id, track, strategy: name, cached: true });\n          continue;\n        }\n\n        options.onCell?.({ caseId: c.id, track, strategy: name, cached: false });\n\n        const artifacts =\n          c.artifactsByTrack?.[track] ?? (await materializeTrack(c.artifacts, track));\n        const start = Date.now();\n\n        let data: unknown;\n        let usage = emptyUsage();\n        let valid = true;\n        let error: string | undefined;\n\n        try {\n          const result = await extract<unknown>({ artifacts, schema: c.schema as never, strategy });\n          data = result.data;\n          usage = result.usage;\n          if (result.error) {\n            valid = false;\n            error = result.error.message;\n          }\n        } catch (e) {\n          valid = false;\n          error = (e as Error).message;\n          data = null;\n        }\n\n        const cell: CellResult = {\n          caseId: c.id,\n          track,\n          strategy: name,\n          model: modelSpec,\n          cached: false,\n          valid,\n          error,\n          score: scoreData(c.gold, c.transform ? c.transform(data) : data, c.metrics),\n          usage,\n          latencyMs: Date.now() - start,\n          costUsd: estimateCostUsd(modelSpec, usage),\n          data,\n        };\n\n        if (useCache) await cacheSet(cacheDir, cacheKey, cell);\n        cells.push(cell);\n      }\n    }\n  }\n\n  return buildReport(modelSpec, cells, variant);\n}\n\n// ---------------------------------------------------------------------------\n// Report builder\n// ---------------------------------------------------------------------------\n\nexport function buildReport(model: string, cells: CellResult[], variant?: string): BenchmarkReport {\n  const summary = new Map<string, SummaryRow>();\n\n  for (const cell of cells) {\n    const key = `${cell.strategy}|${cell.track}`;\n    let row = summary.get(key);\n    if (!row) {\n      row = {\n        strategy: cell.strategy,\n        track: cell.track,\n        cases: 0,\n        meanF1: 0,\n        meanPrecision: 0,\n        meanRecall: 0,\n        validityRate: 0,\n        exactMatchRate: 0,\n        totalInputTokens: 0,\n        totalOutputTokens: 0,\n        totalCostUsd: 0,\n        meanCostUsd: 0,\n        totalLatencyMs: 0,\n        meanLatencyMs: 0,\n      };\n      summary.set(key, row);\n    }\n    row.cases++;\n    row.meanF1 += cell.score.f1;\n    row.meanPrecision += cell.score.precision;\n    row.meanRecall += cell.score.recall;\n    if (cell.valid) row.validityRate++;\n    if (cell.score.exactMatch) row.exactMatchRate++;\n    row.totalInputTokens += cell.usage.inputTokens;\n    row.totalOutputTokens += cell.usage.outputTokens;\n    row.totalCostUsd += cell.costUsd ?? estimateCostUsd(model, cell.usage);\n    row.totalLatencyMs += cell.latencyMs;\n    row.meanLatencyMs += cell.latencyMs;\n  }\n\n  for (const row of summary.values()) {\n    row.meanF1 /= row.cases;\n    row.meanPrecision /= row.cases;\n    row.meanRecall /= row.cases;\n    row.validityRate /= row.cases;\n    row.exactMatchRate /= row.cases;\n    row.meanCostUsd = row.cases ? row.totalCostUsd / row.cases : 0;\n    row.meanLatencyMs = row.cases ? row.meanLatencyMs / row.cases : 0;\n  }\n\n  return {\n    model,\n    variant,\n    generatedAt: new Date().toISOString(),\n    cells,\n    summary: [...summary.values()],\n  };\n}\n","import { createHash } from \"node:crypto\";\nimport { mkdir, readFile, rename, writeFile } from \"node:fs/promises\";\nimport path from \"node:path\";\n\n/** Total, key-order-insensitive serialization for cache keys. */\nconst stableStringify = (v: unknown): string => {\n  if (v === null) return \"null\";\n  if (Array.isArray(v)) return `[${v.map(stableStringify).join(\",\")}]`;\n  if (typeof v === \"object\") {\n    const rec = v as Record<string, unknown>;\n    const keys = Object.keys(rec).sort();\n    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(rec[k])}`).join(\",\")}}`;\n  }\n  return JSON.stringify(v) ?? String(v);\n};\n\n/** Stable short hash of a cache key. */\nexport const hashKey = (input: unknown): string =>\n  createHash(\"sha256\").update(stableStringify(input)).digest(\"hex\").slice(0, 16);\n\nexport async function cacheGet<T>(dir: string, key: string): Promise<T | undefined> {\n  try {\n    const raw = await readFile(path.join(dir, `${key}.json`), \"utf8\");\n    return JSON.parse(raw) as T;\n  } catch {\n    return undefined;\n  }\n}\n\nexport async function cacheSet(dir: string, key: string, value: unknown): Promise<void> {\n  const file = path.join(dir, `${key}.json`);\n  await mkdir(dir, { recursive: true });\n  const tmp = `${file}.tmp`;\n  await writeFile(tmp, JSON.stringify(value, null, 2), \"utf8\");\n  await rename(tmp, file);\n}\n","import { mkdir, writeFile } from \"node:fs/promises\";\nimport path from \"node:path\";\nimport type { BenchmarkReport } from \"./runner\";\n\nconst pct = (v: number): string => `${(v * 100).toFixed(1)}%`;\n\n/** Render the summary as a Markdown table. */\nexport function toMarkdownTable(report: BenchmarkReport): string {\n  const lines: string[] = [];\n  lines.push(`# Benchmark report`);\n  lines.push(``);\n  lines.push(`- model: \\`${report.model}\\``);\n  if (report.variant) lines.push(`- variant: \\`${report.variant}\\``);\n  lines.push(`- generated: ${report.generatedAt}`);\n  lines.push(``);\n  lines.push(\n    `| strategy | track | cases | F1 | precision | recall | valid | exact | in tok | out tok | cost (USD) | ms/case | total ms |`,\n  );\n  lines.push(`|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|`);\n  for (const row of report.summary) {\n    lines.push(\n      `| ${row.strategy} | ${row.track} | ${row.cases} | ${pct(row.meanF1)} | ${pct(row.meanPrecision)} | ${pct(row.meanRecall)} | ${pct(row.validityRate)} | ${pct(row.exactMatchRate)} | ${row.totalInputTokens} | ${row.totalOutputTokens} | ${row.totalCostUsd.toFixed(4)} | ${Math.round(row.meanLatencyMs)} | ${row.totalLatencyMs} |`,\n    );\n  }\n  return lines.join(\"\\n\");\n}\n\n/** Write the full JSON report and a Markdown summary next to it. */\nexport async function saveReport(report: BenchmarkReport, file: string): Promise<void> {\n  await mkdir(path.dirname(file), { recursive: true });\n  await writeFile(file, JSON.stringify(report, null, 2), \"utf8\");\n  const mdFile = file.replace(/\\.json$/, \"\") + \".md\";\n  await writeFile(mdFile, toMarkdownTable(report), \"utf8\");\n}\n"],"mappings":";AAiBO,IAAM,SAA2B;AAAA,EACtC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AA8EO,SAAS,WAAoB,GAAmD;AACrF,SAAO;AACT;;;AC/FO,IAAM,kBAAkB;AAGxB,IAAM,oBAAoB,MAC/B,QAAQ,IAAI,2BAA2B,GAAG,QAAQ,IAAI,QAAQ,GAAG;AAG5D,IAAM,oBAAoB,MAAc,GAAG,kBAAkB,CAAC;;;ACHrE,IAAM,UAAwC;AAAA;AAAA,EAE5C,oDAAoD,EAAE,OAAO,MAAM,QAAQ,KAAK;AAAA;AAAA,EAEhF,iCAAiC,EAAE,OAAO,MAAM,QAAQ,IAAI;AAC9D;AAEA,IAAM,eAAe,MAAgD;AACnE,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,QAAO;AACjB,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF,GAAG;AAGH,IAAM,iBAAiB,CAAC,UAA0B,MAAM,QAAQ,QAAQ,EAAE;AAEnE,SAAS,WAAW,OAAyC;AAClE,QAAM,MAAM,eAAe,KAAK;AAChC,SAAO,cAAc,GAAG,KAAK,QAAQ,GAAG;AAC1C;AAGO,SAAS,gBAAgB,OAAe,OAAsB;AACnE,QAAM,IAAI,WAAW,KAAK;AAC1B,MAAI,CAAC,EAAG,QAAO;AACf,SAAQ,MAAM,cAAc,MAAa,EAAE,QAAS,MAAM,eAAe,MAAa,EAAE;AAC1F;;;ACzCA,SAAS,gBAAgB;AAKlB,IAAM,oBAAoB,CAC/B,UACqD;AACrD,UAAQ,OAAO;AAAA,IACb,KAAK;AACH,aAAO,EAAE,eAAe,OAAO,aAAa,MAAM;AAAA,IACpD,KAAK;AACH,aAAO,EAAE,eAAe,MAAM,aAAa,MAAM;AAAA,IACnD,KAAK;AACH,aAAO,EAAE,eAAe,OAAO,aAAa,KAAK;AAAA,IACnD,KAAK;AACH,aAAO,EAAE,eAAe,MAAM,aAAa,KAAK;AAAA,EACpD;AACF;AASA,eAAsB,iBAAiB,WAAuB,OAAmC;AAC/F,SAAO,QAAQ;AAAA,IACb,UAAU,IAAI,OAAO,aAAa;AAChC,UAAI,SAAS,SAAS,MAAO,QAAO;AACpC,YAAM,SAAS,MAAM,SAAS,IAAI;AAClC,aAAO,SAAS,QAAQ,kBAAkB,KAAK,CAAC;AAAA,IAClD,CAAC;AAAA,EACH;AACF;;;AChCO,IAAM,QAAQ,uBAAO,2BAA2B;AAIvD,IAAM,kBAAkB,CAAC,MAAsB,EAAE,UAAU,KAAK,EAAE,QAAQ,oBAAoB,EAAE;AAGzF,IAAM,kBAAkB,CAAC,MAC9B,gBAAgB,EAAE,YAAY,CAAC,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAGtD,IAAM,sBAAsB,CAAC,MAClC,gBAAgB,EAAE,YAAY,CAAC,EAAE,QAAQ,eAAe,EAAE;AAOrD,SAAS,eAAe,OAAgB,QAAiC;AAC9E,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAElD,MAAI,WAAW,SAAS;AACtB,QAAI,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,WAAW;AACxF,aAAO;AAAA,IACT;AACA,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B;AAEA,MAAI,WAAW,YAAY;AACzB,QAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAW,QAAO;AACpE,WAAO,oBAAoB,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK,CAAC;AAAA,EACtF;AAGA,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAW,QAAO;AACpE,SAAO,gBAAgB,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK,CAAC;AAClF;AAEA,IAAM,qBAAqB;AAMpB,SAAS,YAAY,GAAY,GAAY,QAA8B;AAChF,QAAM,KAAK,eAAe,GAAG,MAAM;AACnC,QAAM,KAAK,eAAe,GAAG,MAAM;AAEnC,MAAI,OAAO,SAAS,OAAO,MAAO,QAAO,OAAO,SAAS,OAAO;AAEhE,MAAI,WAAW,eAAe,OAAO,OAAO,YAAY,OAAO,OAAO,UAAU;AAC9E,UAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC;AACpD,WAAO,KAAK,IAAI,KAAK,EAAE,KAAK,qBAAqB;AAAA,EACnD;AAEA,SAAO,OAAO;AAChB;;;ACpDA,IAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,gBAAgB,CAAC,MAAwB;AAC7C,MAAI,OAAO,MAAM,SAAU,QAAO,CAAC;AACnC,SAAO,MAAM;AAAA,IACX,IAAI;AAAA,MACF,EACG,YAAY,EACZ,MAAM,iBAAiB,EACvB,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC;AAAA,IACrD;AAAA,EACF;AACF;AAEA,SAAS,WAAW,MAAe,MAA2B;AAC5D,QAAM,IAAI,MAAM;AAChB,QAAM,KAAK,cAAc,OAAO,QAAQ,EAAE,CAAC;AAC3C,QAAM,KAAK,cAAc,OAAO,QAAQ,EAAE,CAAC;AAC3C,QAAM,KAAK,IAAI,IAAI,EAAE;AACrB,QAAM,KAAK,IAAI,IAAI,EAAE;AACrB,QAAM,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,EAAE;AACvC,QAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE;AACxC,QAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE;AACxC,IAAE,KAAK;AACP,IAAE,KAAK;AACP,IAAE,KAAK;AACP,IAAE,YAAY,GAAG;AACjB,IAAE,YAAY,GAAG;AACjB,SAAO,SAAS,CAAC;AACnB;AAgCA,IAAM,QAAQ,OAAmB;AAAA,EAC/B,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,WAAW;AAAA,EACX,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,IAAI;AAAA,EACJ,YAAY;AAAA,EACZ,aAAa,CAAC;AAChB;AAEA,IAAM,WAAW,CAAC,MAA8B;AAC9C,IAAE,YAAY,EAAE,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE;AACvD,IAAE,SAAS,EAAE,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE;AACpD,IAAE,KAAK,EAAE,YAAY,EAAE,WAAW,IAAI,IAAK,IAAI,EAAE,YAAY,EAAE,UAAW,EAAE,YAAY,EAAE;AAC1F,IAAE,aAAa,EAAE,OAAO,KAAK,EAAE,OAAO;AACtC,SAAO;AACT;AAEA,IAAM,QAAQ,CAAC,MAAkB,UAA4B;AAC3D,OAAK,MAAM,MAAM;AACjB,OAAK,MAAM,MAAM;AACjB,OAAK,MAAM,MAAM;AACjB,OAAK,aAAa,MAAM;AACxB,OAAK,aAAa,MAAM;AACxB,OAAK,YAAY,KAAK,GAAG,MAAM,WAAW;AAC5C;AAEA,IAAM,SAAS,CAAC,MACd,MAAM,QACN,MAAM,UACN,OAAO,MAAM,YACb,OAAO,MAAM,YACb,OAAO,MAAM;AAEf,IAAM,WAAW,CAAC,MAChB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AAGlD,IAAM,YAAY,CAAC,MAAuB;AAC/C,MAAI,MAAM,QAAQ,CAAC,EAAG,QAAO,EAAE,OAAe,CAAC,GAAG,SAAS,IAAI,UAAU,IAAI,GAAG,CAAC;AACjF,MAAI,SAAS,CAAC,GAAG;AACf,WAAO,OAAO,OAAO,CAAC,EAAE,OAAe,CAAC,GAAG,SAAS,IAAI,UAAU,IAAI,GAAG,CAAC;AAAA,EAC5E;AACA,SAAO;AACT;AAGA,IAAM,YAAY,CAACA,UAAyBA,MAAK,QAAQ,eAAe,IAAI;AAE5E,IAAM,YAAY,CAAC,MAAuBA;AAAA;AAAA;AAAA,EAGxC,KAAK,SAASA,MAAK,MAAM,GAAG,EAAE,IAAI,CAAW,KAAK,KAAK,WAAW;AAAA;AAEpE,IAAM,eAAe,CAAC,MAAuBA,UAC3C,KAAK,QAAQ,KAAK,CAAC,MAAM,UAAU,EAAE,IAAI,MAAM,UAAUA,KAAI,CAAC,KAC9D,KAAK,QAAQ,KAAK,CAAC,MAAM,UAAUA,KAAI,EAAE,SAAS,UAAU,EAAE,IAAI,CAAC,CAAC;AAEtE,IAAM,YAAY,CAAC,MAAuBA,WACvC,KAAK,QAAQ,CAAC,GAAG;AAAA,EAChB,CAAC,MAAM,UAAU,CAAC,MAAM,UAAUA,KAAI,KAAK,UAAUA,KAAI,EAAE,SAAS,UAAU,CAAC,CAAC;AAClF;AAEF,SAAS,WAAW,OAAkB,KAAgD;AACpF,QAAM,MAAM,oBAAI,IAAuB;AACvC,QAAM,OAAO,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,GAAG;AAC5C,aAAW,QAAQ,OAAO;AACxB,UAAM,IAAI,KACP,IAAI,CAAC,OAAQ,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,MAAU,EACnD,IAAI,CAAC,MAAM,OAAO,eAAe,GAAG,YAAY,CAAC,CAAC,EAClD,KAAK,GAAG;AACX,UAAM,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC;AAC3B,QAAI,KAAK,IAAI;AACb,QAAI,IAAI,GAAG,GAAG;AAAA,EAChB;AACA,SAAO;AACT;AAGA,SAAS,UAAUA,OAAc,MAAe,MAAe,MAAmC;AAChG,QAAM,SAAS,UAAU,MAAMA,KAAI;AACnC,MAAI,WAAW,YAAY,OAAO,SAAS,YAAY,OAAO,SAAS,WAAW;AAChF,WAAO,WAAW,MAAM,IAAI;AAAA,EAC9B;AACA,QAAM,IAAI,MAAM;AAChB,IAAE,YAAY,OAAO,IAAI,IAAI,IAAI,UAAU,IAAI;AAC/C,IAAE,YAAY,OAAO,IAAI,IAAI,IAAI,UAAU,IAAI;AAE/C,MAAI,YAAY,MAAM,MAAM,MAAM,GAAG;AACnC,MAAE,KAAK,OAAO,IAAI,IAAI,IAAI,UAAU,IAAI;AAExC,WAAO,SAAS,CAAC;AAAA,EACnB;AAEA,QAAM,YAAY,SAAS,QAAQ,SAAS;AAC5C,QAAM,YAAY,SAAS,QAAQ,SAAS;AAC5C,MAAI,UAAW,GAAE,KAAK,EAAE;AAAA,WACf,UAAW,GAAE,KAAK,EAAE;AAAA,OACxB;AACH,MAAE,KAAK,EAAE;AACT,MAAE,KAAK,EAAE;AAAA,EACX;AACA,IAAE,YAAY,KAAK,EAAE,MAAAA,OAAM,MAAM,MAAM,OAAO,CAAC;AAC/C,SAAO,SAAS,CAAC;AACnB;AAEA,SAAS,aAAaA,OAAc,GAAc,GAAc,MAAmC;AACjG,QAAM,IAAI,MAAM;AAChB,QAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,UAAM,GAAG,QAAQ,GAAGA,KAAI,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC,KAAK,MAAM,IAAI,CAAC;AAAA,EACrE;AACA,SAAO,SAAS,CAAC;AACnB;AAEA,SAAS,aACPA,OACA,GACA,GACA,KACA,MACY;AACZ,QAAM,IAAI,MAAM;AAChB,QAAM,KAAK,WAAW,GAAG,GAAG;AAC5B,QAAM,KAAK,WAAW,GAAG,GAAG;AAM5B,MAAI,aAAa;AAEjB,aAAW,CAAC,GAAG,MAAM,KAAK,IAAI;AAC5B,UAAM,SAAS,GAAG,IAAI,CAAC;AACvB,QAAI,CAAC,UAAU,OAAO,WAAW,GAAG;AAClC,iBAAW,MAAM,QAAQ;AACvB,UAAE,MAAM,UAAU,EAAE;AACpB,UAAE,aAAa,UAAU,EAAE;AAAA,MAC7B;AACA;AAAA,IACF;AACA,iBAAa;AACb,UAAM,QAAQ,KAAK,IAAI,OAAO,QAAQ,OAAO,MAAM;AACnD,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,GAAG,QAAQ,GAAGA,KAAI,IAAI,OAAO,CAAC,CAAC,KAAK,OAAO,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC;AAAA,IACvE;AACA,aAAS,IAAI,OAAO,IAAI,OAAO,QAAQ,KAAK;AAC1C,QAAE,MAAM,UAAU,OAAO,CAAC,CAAE;AAC5B,QAAE,aAAa,UAAU,OAAO,CAAC,CAAE;AAAA,IACrC;AACA,aAAS,IAAI,OAAO,IAAI,OAAO,QAAQ,KAAK;AAC1C,QAAE,MAAM,UAAU,OAAO,CAAC,CAAE;AAC5B,QAAE,aAAa,UAAU,OAAO,CAAC,CAAE;AAAA,IACrC;AAAA,EACF;AAEA,aAAW,CAAC,GAAG,MAAM,KAAK,IAAI;AAC5B,QAAI,GAAG,IAAI,CAAC,EAAG;AACf,eAAW,MAAM,QAAQ;AACvB,QAAE,MAAM,UAAU,EAAE;AACpB,QAAE,aAAa,UAAU,EAAE;AAAA,IAC7B;AAAA,EACF;AAEA,MAAI,CAAC,cAAc,EAAE,SAAS,KAAK,EAAE,SAAS,GAAG;AAC/C,WAAO,aAAaA,OAAM,GAAG,GAAG,IAAI;AAAA,EACtC;AAEA,SAAO,SAAS,CAAC;AACnB;AAEA,SAAS,SAASA,OAAc,GAAc,GAAc,MAAmC;AAE7F,QAAM,IAAI,MAAM;AAChB,QAAM,YAAY,oBAAI,IAAoB;AAC1C,QAAM,YAAY,oBAAI,IAAoB;AAC1C,QAAM,SAAS,UAAU,MAAM,UAAUA,KAAI,CAAC;AAC9C,QAAM,OAAO,CAAC,KAA0B,MAAe;AACrD,UAAM,IAAI,OAAO,eAAe,GAAG,MAAM,CAAC;AAC1C,QAAI,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,EAClC;AAEA,aAAW,KAAK,EAAG,MAAK,WAAW,CAAC;AACpC,aAAW,KAAK,EAAG,MAAK,WAAW,CAAC;AAEpC,IAAE,YAAY,EAAE;AAChB,IAAE,YAAY,EAAE;AAEhB,aAAW,CAAC,GAAG,EAAE,KAAK,WAAW;AAC/B,UAAM,KAAK,UAAU,IAAI,CAAC,KAAK;AAC/B,UAAM,SAAS,KAAK,IAAI,IAAI,EAAE;AAC9B,MAAE,MAAM;AACR,MAAE,MAAM,KAAK;AACb,MAAE,MAAM,KAAK;AACb,QAAI,SAAS,IAAI;AACf,QAAE,YAAY,KAAK;AAAA,QACjB,MAAAA;AAAA,QACA,MAAM,EAAE,KAAK,CAAC,MAAM,OAAO,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;AAAA,QAC3D,MAAM;AAAA,QACN;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,aAAW,CAAC,GAAG,EAAE,KAAK,WAAW;AAC/B,UAAM,KAAK,UAAU,IAAI,CAAC,KAAK;AAC/B,QAAI,KAAK,IAAI;AACX,QAAE,YAAY,KAAK;AAAA,QACjB,MAAAA;AAAA,QACA,MAAM;AAAA,QACN,MAAM,EAAE,KAAK,CAAC,MAAM,OAAO,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;AAAA,QAC3D;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO,SAAS,CAAC;AACnB;AAEA,SAAS,QAAQA,OAAc,MAAe,MAAe,MAAmC;AAC9F,MAAI,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG;AAChC,WAAO,UAAUA,OAAM,MAAM,MAAM,IAAI;AAAA,EACzC;AAEA,MAAI,MAAM,QAAQ,IAAI,KAAK,MAAM,QAAQ,IAAI,GAAG;AAC9C,UAAM,IAAI,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC,IAAI;AAC5C,UAAM,IAAI,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC,IAAI;AAC5C,QAAI,UAAU,MAAMA,KAAI,GAAG;AACzB,aAAO,SAASA,OAAM,GAAG,GAAG,IAAI;AAAA,IAClC;AACA,UAAM,QAAQ,aAAa,MAAMA,KAAI;AACrC,WAAO,QAAQ,aAAaA,OAAM,GAAG,GAAG,MAAM,KAAK,IAAI,IAAI,aAAaA,OAAM,GAAG,GAAG,IAAI;AAAA,EAC1F;AAEA,QAAM,IAAI,MAAM;AAChB,QAAM,OAAO,oBAAI,IAAI,CAAC,GAAG,OAAO,KAAK,IAAc,GAAG,GAAG,OAAO,KAAK,IAAc,CAAC,CAAC;AACrF,aAAW,OAAO,MAAM;AACtB,UAAM,YAAYA,UAAS,KAAK,MAAM,GAAGA,KAAI,IAAI,GAAG;AACpD,UAAM,KAAM,KAAiC,GAAG,KAAK;AACrD,UAAM,KAAM,KAAiC,GAAG,KAAK;AACrD,UAAM,GAAG,QAAQ,WAAW,IAAI,IAAI,IAAI,CAAC;AAAA,EAC3C;AACA,SAAO,SAAS,CAAC;AACnB;AAKO,SAAS,UAAU,MAAe,MAAe,OAAwB,CAAC,GAAe;AAC9F,SAAO,QAAQ,IAAI,MAAM,MAAM,IAAI;AACrC;;;ACvYA,SAAS,SAAS,cAAc,oBAAoB;AACpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;;;ACXP,SAAS,kBAAkB;AAC3B,SAAS,OAAO,UAAU,QAAQ,iBAAiB;AACnD,OAAO,UAAU;AAGjB,IAAM,kBAAkB,CAAC,MAAuB;AAC9C,MAAI,MAAM,KAAM,QAAO;AACvB,MAAI,MAAM,QAAQ,CAAC,EAAG,QAAO,IAAI,EAAE,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACjE,MAAI,OAAO,MAAM,UAAU;AACzB,UAAM,MAAM;AACZ,UAAM,OAAO,OAAO,KAAK,GAAG,EAAE,KAAK;AACnC,WAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AAAA,EACzF;AACA,SAAO,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC;AACtC;AAGO,IAAM,UAAU,CAAC,UACtB,WAAW,QAAQ,EAAE,OAAO,gBAAgB,KAAK,CAAC,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AAE/E,eAAsB,SAAY,KAAa,KAAqC;AAClF,MAAI;AACF,UAAM,MAAM,MAAM,SAAS,KAAK,KAAK,KAAK,GAAG,GAAG,OAAO,GAAG,MAAM;AAChE,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,SAAS,KAAa,KAAa,OAA+B;AACtF,QAAM,OAAO,KAAK,KAAK,KAAK,GAAG,GAAG,OAAO;AACzC,QAAM,MAAM,KAAK,EAAE,WAAW,KAAK,CAAC;AACpC,QAAM,MAAM,GAAG,IAAI;AACnB,QAAM,UAAU,KAAK,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAC3D,QAAM,OAAO,KAAK,IAAI;AACxB;;;AD2BA,IAAM,WAAiD;AAAA,EACrD,QAAQ,CAAC,OAAO,OAAO,iBAAiB,OAAO,EAAE,OAAO,oBAAoB,aAAa,CAAC;AAAA,EAC1F,UAAU,CAAC,OAAO,OAAO,iBACvB,SAAS,EAAE,OAAO,YAAY,OAAO,WAAW,KAAQ,oBAAoB,aAAa,CAAC;AAAA,EAC5F,YAAY,CAAC,OAAO,OAAO,iBACzB,WAAW,EAAE,OAAO,WAAW,KAAQ,oBAAoB,aAAa,CAAC;AAAA,EAC3E,mBAAmB,CAAC,OAAO,OAAO,iBAChC,kBAAkB;AAAA,IAChB;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,IACX,oBAAoB;AAAA,EACtB,CAAC;AAAA,EACH,qBAAqB,CAAC,OAAO,OAAO,iBAClC,oBAAoB;AAAA,IAClB;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,IACX,oBAAoB;AAAA,EACtB,CAAC;AAAA,EACH,YAAY,CAAC,OAAO,OAAO,iBACzB,WAAW,EAAE,OAAO,YAAY,OAAO,WAAW,KAAQ,oBAAoB,aAAa,CAAC;AAAA,EAC9F,qBAAqB,CAAC,OAAO,OAAO,iBAClC,oBAAoB;AAAA,IAClB;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,IACX,oBAAoB;AAAA,EACtB,CAAC;AAAA,EACH,OAAO,CAAC,QAAQ,WAAW,iBAAiB;AAC1C,UAAM,CAAC,UAAU,GAAG,IAAI,IAAI,UAAU,MAAM,GAAG;AAC/C,UAAM,UAAU,KAAK,KAAK,GAAG;AAC7B,QAAI,CAAC,YAAY,CAAC;AAChB,YAAM,IAAI,MAAM,yCAAyC,SAAS,EAAE;AACtE,WAAO,MAAM;AAAA,MACX;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,eAAe;AAAA,MACf,QAAQ;AAAA,MACR,oBAAoB;AAAA,IACtB,CAAC;AAAA,EACH;AACF;AAqEA,SAAS,gBACP,OACA,OACA,WACyD;AACzD,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,UAAU,SAAS,KAAK;AAC9B,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI;AAAA,QACR,6BAA6B,KAAK,gBAAgB,OAAO,KAAK,QAAQ,EAAE,KAAK,IAAI,CAAC;AAAA,MACpF;AAAA,IACF;AACA,WAAO,EAAE,MAAM,OAAO,UAAU,QAAQ,OAAO,SAAS,EAAE;AAAA,EAC5D;AAEA,MAAI,OAAO,UAAU,YAAY;AAC/B,UAAM,IAAI,MAAM,OAAO,SAAS;AAChC,WAAO,EAAE,MAAM,EAAE,MAAM,UAAU,EAAE;AAAA,EACrC;AAEA,MAAI,cAAc,OAAO;AACvB,UAAM,QAAQ,MAAM;AACpB,QAAI,OAAO,UAAU,UAAU;AAC7B,YAAM,UAAU,SAAS,KAAK;AAC9B,UAAI,CAAC,QAAS,OAAM,IAAI,MAAM,6BAA6B,KAAK,EAAE;AAClE,aAAO;AAAA,QACL,MAAM,MAAM,SAAS;AAAA,QACrB,UAAU,QAAQ,OAAO,WAAW,MAAM,YAAY;AAAA,MACxD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,OAAO,WAAW,MAAM,YAAY;AACpD,WAAO,EAAE,MAAM,MAAM,SAAS,EAAE,MAAM,UAAU,EAAE;AAAA,EACpD;AAEA,SAAO;AAAA,IACL,MAAO,MAAsC;AAAA,IAC7C,UAAU;AAAA,EACZ;AACF;AAEA,IAAM,aAAa,OAAc,EAAE,aAAa,GAAG,cAAc,GAAG,aAAa,EAAE;AAMnF,eAAsB,aAAa,SAA+C;AAChF,QAAM,YAAY,QAAQ,SAAS;AACnC,QAAM,QAAQ,MAAM,aAAa,SAAS;AAC1C,QAAM,WAAW,QAAQ,YAAY,kBAAkB;AACvD,QAAM,WAAW,QAAQ,SAAS;AAClC,QAAM,UAAU,QAAQ,WAAW;AAEnC,QAAM,SAAS,QAAQ,WAAW,IAAI,CAAC,MAAM,gBAAgB,GAAG,OAAO,SAAS,CAAC;AACjF,QAAM,QAAsB,CAAC;AAE7B,aAAW,KAAK,QAAQ,OAAO;AAC7B,UAAM,SAAS,EAAE,UAAU,CAAC,GAAG,MAAM;AACrC,eAAW,SAAS,QAAQ;AAC1B,iBAAW,EAAE,MAAM,SAAS,KAAK,QAAQ;AACvC,cAAM,WAAW,QAAQ;AAAA,UACvB,QAAQ,EAAE;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP;AAAA,UACA,QAAQ,aAAa,EAAE,MAAM;AAAA,UAC7B,MAAM,EAAE;AAAA,QACV,CAAC;AAED,cAAM,SAAS,WAAW,MAAM,SAAqB,UAAU,QAAQ,IAAI;AAC3E,YAAI,QAAQ;AACV,iBAAO,SAAS;AAEhB,cAAI,OAAO,YAAY;AACrB,mBAAO,UAAU,gBAAgB,WAAW,OAAO,KAAK;AAC1D,gBAAM,KAAK,MAAM;AACjB,kBAAQ,SAAS,EAAE,QAAQ,EAAE,IAAI,OAAO,UAAU,MAAM,QAAQ,KAAK,CAAC;AACtE;AAAA,QACF;AAEA,gBAAQ,SAAS,EAAE,QAAQ,EAAE,IAAI,OAAO,UAAU,MAAM,QAAQ,MAAM,CAAC;AAEvE,cAAM,YACJ,EAAE,mBAAmB,KAAK,KAAM,MAAM,iBAAiB,EAAE,WAAW,KAAK;AAC3E,cAAM,QAAQ,KAAK,IAAI;AAEvB,YAAI;AACJ,YAAI,QAAQ,WAAW;AACvB,YAAI,QAAQ;AACZ,YAAI;AAEJ,YAAI;AACF,gBAAM,SAAS,MAAM,QAAiB,EAAE,WAAW,QAAQ,EAAE,QAAiB,SAAS,CAAC;AACxF,iBAAO,OAAO;AACd,kBAAQ,OAAO;AACf,cAAI,OAAO,OAAO;AAChB,oBAAQ;AACR,oBAAQ,OAAO,MAAM;AAAA,UACvB;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ;AACR,kBAAS,EAAY;AACrB,iBAAO;AAAA,QACT;AAEA,cAAM,OAAmB;AAAA,UACvB,QAAQ,EAAE;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,IAAI,IAAI,MAAM,EAAE,OAAO;AAAA,UAC1E;AAAA,UACA,WAAW,KAAK,IAAI,IAAI;AAAA,UACxB,SAAS,gBAAgB,WAAW,KAAK;AAAA,UACzC;AAAA,QACF;AAEA,YAAI,SAAU,OAAM,SAAS,UAAU,UAAU,IAAI;AACrD,cAAM,KAAK,IAAI;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,YAAY,WAAW,OAAO,OAAO;AAC9C;AAMO,SAAS,YAAY,OAAe,OAAqB,SAAmC;AACjG,QAAM,UAAU,oBAAI,IAAwB;AAE5C,aAAW,QAAQ,OAAO;AACxB,UAAM,MAAM,GAAG,KAAK,QAAQ,IAAI,KAAK,KAAK;AAC1C,QAAI,MAAM,QAAQ,IAAI,GAAG;AACzB,QAAI,CAAC,KAAK;AACR,YAAM;AAAA,QACJ,UAAU,KAAK;AAAA,QACf,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,eAAe;AAAA,QACf,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,gBAAgB;AAAA,QAChB,kBAAkB;AAAA,QAClB,mBAAmB;AAAA,QACnB,cAAc;AAAA,QACd,aAAa;AAAA,QACb,gBAAgB;AAAA,QAChB,eAAe;AAAA,MACjB;AACA,cAAQ,IAAI,KAAK,GAAG;AAAA,IACtB;AACA,QAAI;AACJ,QAAI,UAAU,KAAK,MAAM;AACzB,QAAI,iBAAiB,KAAK,MAAM;AAChC,QAAI,cAAc,KAAK,MAAM;AAC7B,QAAI,KAAK,MAAO,KAAI;AACpB,QAAI,KAAK,MAAM,WAAY,KAAI;AAC/B,QAAI,oBAAoB,KAAK,MAAM;AACnC,QAAI,qBAAqB,KAAK,MAAM;AACpC,QAAI,gBAAgB,KAAK,WAAW,gBAAgB,OAAO,KAAK,KAAK;AACrE,QAAI,kBAAkB,KAAK;AAC3B,QAAI,iBAAiB,KAAK;AAAA,EAC5B;AAEA,aAAW,OAAO,QAAQ,OAAO,GAAG;AAClC,QAAI,UAAU,IAAI;AAClB,QAAI,iBAAiB,IAAI;AACzB,QAAI,cAAc,IAAI;AACtB,QAAI,gBAAgB,IAAI;AACxB,QAAI,kBAAkB,IAAI;AAC1B,QAAI,cAAc,IAAI,QAAQ,IAAI,eAAe,IAAI,QAAQ;AAC7D,QAAI,gBAAgB,IAAI,QAAQ,IAAI,gBAAgB,IAAI,QAAQ;AAAA,EAClE;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAAA,IACA,SAAS,CAAC,GAAG,QAAQ,OAAO,CAAC;AAAA,EAC/B;AACF;;;AE3WA,SAAS,SAAAC,QAAO,aAAAC,kBAAiB;AACjC,OAAOC,WAAU;AAGjB,IAAM,MAAM,CAAC,MAAsB,IAAI,IAAI,KAAK,QAAQ,CAAC,CAAC;AAGnD,SAAS,gBAAgB,QAAiC;AAC/D,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,oBAAoB;AAC/B,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,cAAc,OAAO,KAAK,IAAI;AACzC,MAAI,OAAO,QAAS,OAAM,KAAK,gBAAgB,OAAO,OAAO,IAAI;AACjE,QAAM,KAAK,gBAAgB,OAAO,WAAW,EAAE;AAC/C,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ;AAAA,EACF;AACA,QAAM,KAAK,kEAAkE;AAC7E,aAAW,OAAO,OAAO,SAAS;AAChC,UAAM;AAAA,MACJ,KAAK,IAAI,QAAQ,MAAM,IAAI,KAAK,MAAM,IAAI,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,aAAa,CAAC,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,IAAI,gBAAgB,MAAM,IAAI,iBAAiB,MAAM,IAAI,aAAa,QAAQ,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,aAAa,CAAC,MAAM,IAAI,cAAc;AAAA,IACpU;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAGA,eAAsB,WAAW,QAAyB,MAA6B;AACrF,QAAMF,OAAME,MAAK,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACnD,QAAMD,WAAU,MAAM,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,MAAM;AAC7D,QAAM,SAAS,KAAK,QAAQ,WAAW,EAAE,IAAI;AAC7C,QAAMA,WAAU,QAAQ,gBAAgB,MAAM,GAAG,MAAM;AACzD;","names":["path","mkdir","writeFile","path"]}

package/dist/pricing.d.ts

@@ -0,0 +1,15 @@
+import type { Usage } from "@struktur/sdk";
+/**
+ * Token pricing in USD per 1M tokens, keyed by the benchmark model string
+ * (`provider/model`). Prices are snapshot baselines; the harness, not the
+ * model, is what's measured, so a small table covering the pinned models is
+ * enough. Override with STRUKTUR_BENCHMARK_PRICES (JSON: `{ "<model>": { input, output } }`).
+ */
+export type ModelPricing = {
+    input: number;
+    output: number;
+};
+export declare function pricingFor(model: string): ModelPricing | undefined;
+/** Estimated USD cost of a token usage for a model. 0 when pricing is unknown. */
+export declare function estimateCostUsd(model: string, usage: Usage): number;
+//# sourceMappingURL=pricing.d.ts.map

package/dist/pricing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../src/pricing.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAE3C;;;;;GAKG;AAEH,MAAM,MAAM,YAAY,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAsB7D,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAGlE;AAED,kFAAkF;AAClF,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,CAInE"}

package/dist/report.d.ts

@@ -0,0 +1,6 @@
+import type { BenchmarkReport } from "./runner";
+/** Render the summary as a Markdown table. */
+export declare function toMarkdownTable(report: BenchmarkReport): string;
+/** Write the full JSON report and a Markdown summary next to it. */
+export declare function saveReport(report: BenchmarkReport, file: string): Promise<void>;
+//# sourceMappingURL=report.d.ts.map

package/dist/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAIhD,8CAA8C;AAC9C,wBAAgB,eAAe,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAkB/D;AAED,oEAAoE;AACpE,wBAAsB,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKrF"}

package/dist/runner.d.ts

@@ -0,0 +1,81 @@
+import type { ExtractionStrategy, Usage } from "@struktur/sdk";
+import type { BenchmarkCase, Track } from "./types";
+import { type FieldScore } from "./scoring/score";
+type BuiltinName = "simple" | "parallel" | "sequential" | "parallelAutoMerge" | "sequentialAutoMerge" | "doublePass" | "doublePassAutoMerge" | "agent";
+/**
+ * Strategy factory: the runner resolves the model once and injects it.
+ * `instructions` maps to `outputInstructions` — accept it if you want prompt
+ * variants, otherwise ignore it.
+ */
+export type StrategyFactory<T = unknown> = (model: unknown, modelSpec: string, instructions?: string) => ExtractionStrategy<T>;
+/** A built-in strategy plus an optional prompt variant and report label. */
+export type StrategyEntry = {
+    label?: string;
+    instructions?: string;
+    strategy: BuiltinName | StrategyFactory;
+};
+export type StrategyLike = BuiltinName | StrategyFactory | ExtractionStrategy<unknown> | StrategyEntry;
+export type RunOptions = {
+    cases: BenchmarkCase[];
+    strategies: StrategyLike[];
+    /** Model string in `provider/model` form. Defaults to the pinned benchmark model. */
+    model?: string;
+    /** Cache directory for result cells. Defaults to `~/.struktur/benchmarks/results`. */
+    cacheDir?: string;
+    /** Set to `false` to skip the disk cache. */
+    cache?: boolean;
+    /** Opaque variant tag included in the cache key — bump to invalidate. */
+    variant?: string;
+    onCell?: (info: {
+        caseId: string;
+        track: Track;
+        strategy: string;
+        cached: boolean;
+    }) => void;
+};
+export type CellResult = {
+    caseId: string;
+    track: Track;
+    strategy: string;
+    model: string;
+    cached: boolean;
+    valid: boolean;
+    error?: string;
+    score: FieldScore;
+    usage: Usage;
+    latencyMs: number;
+    /** Estimated USD cost of the cell's token usage (0 if pricing unknown). */
+    costUsd?: number;
+    /** Raw extracted data (null if extraction failed). */
+    data: unknown;
+};
+export type SummaryRow = {
+    strategy: string;
+    track: Track;
+    cases: number;
+    meanF1: number;
+    meanPrecision: number;
+    meanRecall: number;
+    validityRate: number;
+    exactMatchRate: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    /** Sum of cell costs across the row (USD). */
+    totalCostUsd: number;
+    /** Mean cost per case (USD). */
+    meanCostUsd: number;
+    /** Total wall-clock latency across all cells in the row (ms). */
+    totalLatencyMs: number;
+    meanLatencyMs: number;
+};
+export type BenchmarkReport = {
+    model: string;
+    variant?: string;
+    generatedAt: string;
+    cells: CellResult[];
+    summary: SummaryRow[];
+};
+export declare function runBenchmark(options: RunOptions): Promise<BenchmarkReport>;
+export declare function buildReport(model: string, cells: CellResult[], variant?: string): BenchmarkReport;
+export {};
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAY/D,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAGpD,OAAO,EAAa,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAS7D,KAAK,WAAW,GACZ,QAAQ,GACR,UAAU,GACV,YAAY,GACZ,mBAAmB,GACnB,qBAAqB,GACrB,YAAY,GACZ,qBAAqB,GACrB,OAAO,CAAC;AAEZ;;;;GAIG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI,CACzC,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,YAAY,CAAC,EAAE,MAAM,KAClB,kBAAkB,CAAC,CAAC,CAAC,CAAC;AAE3B,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,WAAW,GAAG,eAAe,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,eAAe,GACf,kBAAkB,CAAC,OAAO,CAAC,GAC3B,aAAa,CAAC;AAuDlB,MAAM,MAAM,UAAU,GAAG;IACvB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;CAC9F,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,UAAU,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,IAAI,EAAE,OAAO,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,YAAY,EAAE,MAAM,CAAC;IACrB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB,CAAC;AAoDF,wBAAsB,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,eAAe,CAAC,CAkFhF;AAMD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,eAAe,CAuDjG"}

package/dist/scoring/normalize.d.ts

@@ -0,0 +1,20 @@
+import type { FieldMetric } from "../types";
+/** Sentinel for "no value": null, undefined, and absent keys all normalize here. */
+export declare const EMPTY: unique symbol;
+export type Normalized = string | number | boolean | typeof EMPTY;
+/** Case, whitespace, and diacritic folding for normal string comparison. */
+export declare const normalizeString: (s: string) => string;
+/** Aggressive folding for semantic equivalence: alphanumerics only. */
+export declare const normalizeAggressive: (s: string) => string;
+/**
+ * Normalize a leaf value according to a metric. Leaves are primitives
+ * (string/number/boolean) or null/undefined; anything else is stringified so a
+ * type mismatch still produces a deterministic, non-equal comparison.
+ */
+export declare function normalizeValue(value: unknown, metric: FieldMetric): Normalized;
+/**
+ * Compare two leaf values under a metric. `EMPTY == EMPTY` is always true:
+ * extracting "nothing" where the source has nothing is correct.
+ */
+export declare function valuesEqual(a: unknown, b: unknown, metric: FieldMetric): boolean;
+//# sourceMappingURL=normalize.d.ts.map

package/dist/scoring/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/scoring/normalize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C,oFAAoF;AACpF,eAAO,MAAM,KAAK,eAAsC,CAAC;AAEzD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,KAAK,CAAC;AAIlE,4EAA4E;AAC5E,eAAO,MAAM,eAAe,GAAI,GAAG,MAAM,KAAG,MACkB,CAAC;AAE/D,uEAAuE;AACvE,eAAO,MAAM,mBAAmB,GAAI,GAAG,MAAM,KAAG,MACa,CAAC;AAE9D;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,UAAU,CAkB9E;AAID;;;GAGG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAYhF"}

package/dist/scoring/score.d.ts

@@ -0,0 +1,39 @@
+import type { FieldMetric, FieldMetricSpec } from "../types";
+import { EMPTY } from "./normalize";
+/**
+ * Field-level score. Fields (leaves) are compared individually so omission
+ * (gold field missing) and hallucination (extra predicted field) are counted
+ * separately, and a wrong value counts against both precision and recall.
+ */
+export type FieldScore = {
+    /** Correctly extracted leaf values. */
+    tp: number;
+    /** Predicted leaves that were wrong (hallucinated or wrong value). */
+    fp: number;
+    /** Gold leaves that were wrong (omitted or wrong value). */
+    fn: number;
+    totalGold: number;
+    totalPred: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    /** True when every gold field matched exactly and no field was hallucinated. */
+    exactMatch: boolean;
+    /** Per-field mismatches for debugging. */
+    fieldErrors: FieldError[];
+};
+export type FieldError = {
+    path: string;
+    gold: unknown;
+    pred: unknown;
+    metric: FieldMetric;
+};
+/** Number of leaf values a (sub)tree contains. `{}` has zero; `null` counts as one. */
+export declare const leafCount: (v: unknown) => number;
+/**
+ * Score an extraction output against its gold instance, field by field.
+ */
+export declare function scoreData(gold: unknown, pred: unknown, spec?: FieldMetricSpec): FieldScore;
+/** Export the sentinel so tests/consumers can build explicit EMPTY comparisons. */
+export { EMPTY };
+//# sourceMappingURL=score.d.ts.map

package/dist/scoring/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/scoring/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,WAAW,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC7E,OAAO,EAAE,KAAK,EAA+B,MAAM,aAAa,CAAC;AA8GjE;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,uCAAuC;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,gFAAgF;IAChF,UAAU,EAAE,OAAO,CAAC;IACpB,0CAA0C;IAC1C,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,WAAW,CAAC;CACrB,CAAC;AA0CF,uFAAuF;AACvF,eAAO,MAAM,SAAS,GAAI,GAAG,OAAO,KAAG,MAMtC,CAAC;AAuMF;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,GAAE,eAAoB,GAAG,UAAU,CAE9F;AAED,mFAAmF;AACnF,OAAO,EAAE,KAAK,EAAE,CAAC"}

package/dist/tracks.d.ts

@@ -0,0 +1,16 @@
+import type { Artifact } from "@struktur/sdk";
+import type { Track } from "./types";
+/** Map a track to the PDF parse flags that produce it. */
+export declare const trackToPdfOptions: (track: Track) => {
+    includeImages: boolean;
+    screenshots: boolean;
+};
+/**
+ * Produce the artifact set for a track.
+ *
+ * PDF artifacts are re-parsed from their raw bytes with the track's image
+ * flags (the parser always keeps text). Image/text/file artifacts are returned
+ * unchanged — their media is intrinsic and can't be toggled off.
+ */
+export declare function materializeTrack(artifacts: Artifact[], track: Track): Promise<Artifact[]>;
+//# sourceMappingURL=tracks.d.ts.map

package/dist/tracks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tracks.d.ts","sourceRoot":"","sources":["../src/tracks.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAErC,0DAA0D;AAC1D,eAAO,MAAM,iBAAiB,GAC5B,OAAO,KAAK,KACX;IAAE,aAAa,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAWhD,CAAC;AAEF;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAQ/F"}

package/dist/types.d.ts

@@ -0,0 +1,88 @@
+import type { Artifact, StandardSchema, TypedJSONSchema, Usage } from "@struktur/sdk";
+/**
+ * The four ways a source can be presented to the model.
+ *
+ * Text is always present — the PDF parser extracts text unconditionally. The
+ * track only varies which images accompany it:
+ * - `text`                       → text only
+ * - `text+embedded`              → text + images embedded in the source
+ * - `text+screenshots`           → text + rendered page screenshots
+ * - `text+embedded+screenshots`  → text + both
+ *
+ * For image-only sources (scanned receipts, photos) text is absent and the
+ * image IS the content, so only `text+embedded` is meaningful.
+ */
+export type Track = "text" | "text+embedded" | "text+screenshots" | "text+embedded+screenshots";
+export declare const TRACKS: readonly Track[];
+/**
+ * Infer the output type `T` of a schema:
+ * - Standard Schema v1 (Zod, Valibot, ArkType, …) → its output type
+ * - TypedJSONSchema<T> (tagged plain JSON Schema)      → T
+ * - plain JSON Schema object                            → unknown (no type info carried)
+ */
+export type SchemaOutput<S> = S extends StandardSchema<unknown, infer T> ? T : S extends TypedJSONSchema<infer T> ? T : unknown;
+/** How a single field is compared against its gold value. */
+export type FieldMetric = "exact" | "normalized" | "tolerance" | "semantic" | "prose";
+/** An array path (dot notation) aligned by per-item key field(s) instead of index. */
+export type ArrayAlignment = {
+    /** e.g. `"items"` or `"invoice.items"` */
+    path: string;
+    /** Field(s) within each item used to match items, e.g. `"name"` or `["floor","area"]`. */
+    key: string | string[];
+};
+/**
+ * Per-field scoring rules. Defaults to `normalized` (case/whitespace/diacritic
+ * folding, numeric equality) when nothing is specified.
+ */
+export type FieldMetricSpec = {
+    default?: FieldMetric;
+    /** Dot-path → metric override. e.g. `{ "total": "tolerance" }` */
+    fields?: Record<string, FieldMetric>;
+    /** Arrays matched by a key field rather than position. */
+    arrays?: ArrayAlignment[];
+    /** Array paths scored as order-insensitive sets (uses the def value metric). */
+    sets?: string[];
+};
+/** Provenance metadata attached to a case (license, attribution). */
+export type CaseSourceInfo = {
+    dataset?: string;
+    license?: string;
+    url?: string;
+    attribution?: string;
+};
+/**
+ * The portable unit of the benchmark: one input → one schema → one correct
+ * answer. `gold` is a fully-populated instance of `schema` — the exact JSON a
+ * perfect extraction should return.
+ *
+ * `artifacts` is the base input. For PDF sources the runner re-parses
+ * `artifacts` per track (see `materializeTrack`); for image/text sources that
+ * can't be re-derived, supply `artifactsByTrack` to pin specific inputs.
+ */
+export interface BenchmarkCase<TSchema = unknown, T = SchemaOutput<TSchema>> {
+    id: string;
+    schema: TSchema;
+    gold: T;
+    artifacts: Artifact[];
+    /** Tracks this case can produce. Defaults to all four. */
+    tracks?: Track[];
+    /**
+     * Pre-materialized artifacts per track. Takes precedence over `artifacts`
+     * (which is still required as the fallback/default). Use when a source can't
+     * be re-parsed from its raw bytes, e.g. image-only datasets.
+     */
+    artifactsByTrack?: Partial<Record<Track, Artifact[]>>;
+    /** Per-field scoring rules for this case. */
+    metrics?: FieldMetricSpec;
+    /**
+     * Optional normalizer applied to a prediction before scoring (and before
+     * caching). Use to make gold/prediction shapes symmetric, e.g. collapsing
+     * building+unit features into one set. Gold is NOT passed through this.
+     */
+    transform?: (data: unknown) => unknown;
+    source?: CaseSourceInfo;
+}
+/** Identity helper that preserves schema → gold type inference. */
+export declare function defineCase<TSchema>(c: BenchmarkCase<TSchema>): BenchmarkCase<TSchema>;
+export type { Usage };
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,eAAe,GAAG,kBAAkB,GAAG,2BAA2B,CAAC;AAEhG,eAAO,MAAM,MAAM,EAAE,SAAS,KAAK,EAKzB,CAAC;AAEX;;;;;GAKG;AACH,MAAM,MAAM,YAAY,CAAC,CAAC,IACxB,CAAC,SAAS,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;AAEpG,6DAA6D;AAC7D,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC;AAEtF,sFAAsF;AACtF,MAAM,MAAM,cAAc,GAAG;IAC3B,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,0FAA0F;IAC1F,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAC1B,gFAAgF;IAChF,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,WAAW,aAAa,CAAC,OAAO,GAAG,OAAO,EAAE,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC;IACzE,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,0DAA0D;IAC1D,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IACtD,6CAA6C;IAC7C,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,CAErF;AAED,YAAY,EAAE,KAAK,EAAE,CAAC"}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@struktur/benchmarks",
+  "version": "2.6.0",
+  "license": "FSL-1.1-MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./datasets": {
+      "types": "./dist/datasets/index.d.ts",
+      "import": "./dist/datasets/index.js"
+    }
+  },
+  "bin": {
+    "struktur-benchmarks": "dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@struktur/sdk": "2.6.0"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "tsup": "^8.0.0",
+    "zod": "^4.4.3"
+  },
+  "scripts": {
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "build": "tsup && tsc --project tsconfig.build.json"
+  }
+}