@struktur/benchmarks 2.6.0

package/dist/cli.js

@@ -0,0 +1,961 @@
+#!/usr/bin/env node
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/config.ts
+var config_exports = {};
+__export(config_exports, {
+  BENCHMARK_MODEL: () => BENCHMARK_MODEL,
+  defaultDatasetDir: () => defaultDatasetDir,
+  defaultResultsDir: () => defaultResultsDir
+});
+var BENCHMARK_MODEL, defaultDatasetDir, defaultResultsDir;
+var init_config = __esm({
+  "src/config.ts"() {
+    "use strict";
+    BENCHMARK_MODEL = "openrouter/deepseek/deepseek-v4-flash-vision-exp";
+    defaultDatasetDir = () => process.env.STRUKTUR_BENCHMARKS_DIR ?? `${process.env.HOME ?? "."}/.struktur/benchmarks`;
+    defaultResultsDir = () => `${defaultDatasetDir()}/results`;
+  }
+});
+
+// src/cli.ts
+import { resolve } from "path";
+
+// src/datasets/source.ts
+init_config();
+import { mkdir, readFile, writeFile } from "fs/promises";
+import path from "path";
+var readCachedCases = async (file) => {
+  try {
+    return JSON.parse(await readFile(file, "utf8"));
+  } catch {
+    return void 0;
+  }
+};
+async function loadDataset(dataset, options = {}) {
+  const base = path.join(options.cacheDir ?? defaultDatasetDir(), dataset.id, dataset.version);
+  const casesFile = path.join(base, "cases.json");
+  const rawDir = path.join(base, "raw");
+  const limit = options.limit;
+  const offset = options.offset ?? 0;
+  let cases = options.force ? void 0 : await readCachedCases(casesFile);
+  if (!cases) {
+    if (dataset.source.kind === "manual") {
+      const hint = dataset.source.hint ?? `download the dataset into ${rawDir}`;
+      throw new Error(`Dataset "${dataset.id}" requires manual download. ${hint}`);
+    }
+    await mkdir(rawDir, { recursive: true });
+    await dataset.download(rawDir, options.fetchImpl);
+    cases = await dataset.convert(rawDir);
+    await mkdir(base, { recursive: true });
+    await writeFile(casesFile, JSON.stringify(cases, null, 2), "utf8");
+    await writeFile(
+      path.join(base, "manifest.json"),
+      JSON.stringify(
+        {
+          id: dataset.id,
+          version: dataset.version,
+          license: dataset.license,
+          attribution: dataset.attribution,
+          generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          count: cases.length
+        },
+        null,
+        2
+      ),
+      "utf8"
+    );
+  }
+  return cases.slice(offset, limit !== void 0 ? offset + limit : void 0);
+}
+var DEFAULT_HUB_BASE = "https://datasets-server.huggingface.co";
+async function downloadHubRows(source, fetchImpl = fetch) {
+  const base = source.baseUrl ?? DEFAULT_HUB_BASE;
+  const repo = encodeURIComponent(source.repo);
+  const config = source.config ?? "default";
+  const split = source.split ?? "train";
+  const pageSize = 100;
+  const rows = [];
+  let offset = 0;
+  for (; ; ) {
+    const url = `${base}/rows?dataset=${repo}&config=${config}&split=${split}&offset=${offset}&length=${pageSize}`;
+    const res = await fetchImpl(url);
+    if (!res.ok) {
+      throw new Error(`Hugging Face datasets-server error ${res.status} for ${url}`);
+    }
+    const body = await res.json();
+    const page = body.rows ?? [];
+    if (page.length === 0) break;
+    rows.push(...page.map((r) => r.row));
+    if (page.length < pageSize) break;
+    offset += page.length;
+  }
+  return rows;
+}
+
+// src/datasets/sroie.ts
+var REPO = "darentang/sroie";
+var CONFIG = "sroie";
+var LABELS = [
+  "O",
+  "B-COMPANY",
+  "I-COMPANY",
+  "B-DATE",
+  "I-DATE",
+  "B-ADDRESS",
+  "I-ADDRESS",
+  "B-TOTAL",
+  "I-TOTAL"
+];
+var SCHEMA = {
+  type: "object",
+  properties: {
+    company: { type: ["string", "null"] },
+    date: { type: ["string", "null"] },
+    address: { type: ["string", "null"] },
+    total: { type: ["string", "null"] }
+  },
+  additionalProperties: false
+};
+function reconstructEntities(words, tags) {
+  const out = {
+    company: null,
+    date: null,
+    address: null,
+    total: null
+  };
+  let current = null;
+  let buffer = [];
+  const flush = () => {
+    if (current && buffer.length > 0) {
+      out[current.toLowerCase()] = buffer.join(" ");
+    }
+    current = null;
+    buffer = [];
+  };
+  for (let i = 0; i < words.length; i++) {
+    const tag = LABELS[tags[i] ?? 0] ?? "O";
+    if (tag.startsWith("B-")) {
+      flush();
+      current = tag.slice(2);
+      buffer = [words[i] ?? ""];
+    } else if (tag.startsWith("I-") && current === tag.slice(2)) {
+      buffer.push(words[i] ?? "");
+    } else {
+      flush();
+    }
+  }
+  flush();
+  return out;
+}
+var imageUrl = (imagePath) => `https://huggingface.co/datasets/${REPO}/resolve/main/${imagePath}`;
+var imageArtifact = (id, url) => ({
+  id: `artifact-${id}`,
+  type: "image",
+  raw: async () => Buffer.from(""),
+  contents: [{ media: [{ type: "image", url }] }]
+});
+function sroieToCases(rows) {
+  const cases = [];
+  for (const row of rows) {
+    const words = row.words ?? [];
+    const tags = row.ner_tags ?? [];
+    const imagePath = row.image_path;
+    if (!imagePath) continue;
+    const gold = reconstructEntities(words, tags);
+    const id = row.id ?? imagePath;
+    const url = imageUrl(imagePath);
+    const text = words.join(" ");
+    cases.push({
+      id: `sroie-${id}`,
+      schema: SCHEMA,
+      gold,
+      artifacts: [imageArtifact(id, url)],
+      tracks: ["text", "text+embedded"],
+      artifactsByTrack: {
+        text: [
+          {
+            id: `artifact-${id}-text`,
+            type: "text",
+            raw: async () => Buffer.from(text, "utf8"),
+            contents: [{ text }]
+          }
+        ],
+        "text+embedded": [imageArtifact(id, url)]
+      },
+      metrics: { default: "normalized" },
+      source: {
+        dataset: "sroie",
+        license: "ICDAR 2019 SROIE (research use)",
+        url: "https://github.com/zzzDavid/ICDAR-2019-SROIE",
+        attribution: "ICDAR 2019 SROIE (Huang et al., 2019)"
+      }
+    });
+  }
+  return cases;
+}
+var download = async (rawDir, fetchImpl) => {
+  const { writeFile: writeFile4 } = await import("fs/promises");
+  const path4 = await import("path");
+  const all = [];
+  for (const split of ["train", "test"]) {
+    const rows = await downloadHubRows(
+      { kind: "hub", repo: REPO, config: CONFIG, split },
+      fetchImpl
+    );
+    all.push(...rows);
+  }
+  await writeFile4(path4.join(rawDir, "rows.json"), JSON.stringify(all), "utf8");
+};
+var convert = async (rawDir) => {
+  const { readFile: readFile3 } = await import("fs/promises");
+  const path4 = await import("path");
+  const rows = JSON.parse(await readFile3(path4.join(rawDir, "rows.json"), "utf8"));
+  return sroieToCases(rows);
+};
+var sroie = {
+  id: "sroie",
+  version: "1",
+  source: { kind: "hub", repo: REPO, config: CONFIG },
+  license: "ICDAR 2019 SROIE (research use)",
+  attribution: "ICDAR 2019 SROIE (Huang et al., 2019)",
+  download,
+  convert
+};
+
+// src/types.ts
+var TRACKS = [
+  "text",
+  "text+embedded",
+  "text+screenshots",
+  "text+embedded+screenshots"
+];
+
+// src/index.ts
+init_config();
+
+// src/pricing.ts
+var PRICING = {
+  // OpenRouter: $0.22/M input, $0.66/M output
+  "openrouter/deepseek/deepseek-v4-flash-vision-exp": { input: 0.22, output: 0.66 },
+  // OpenAI via OpenRouter: $0.15/M input, $0.60/M output
+  "openrouter/openai/gpt-4o-mini": { input: 0.15, output: 0.6 }
+};
+var envOverride = (() => {
+  const raw = process.env.STRUKTUR_BENCHMARK_PRICES;
+  if (!raw) return void 0;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+})();
+var normalizeModel = (model) => model.replace(/#.*$/, "");
+function pricingFor(model) {
+  const key = normalizeModel(model);
+  return envOverride?.[key] ?? PRICING[key];
+}
+function estimateCostUsd(model, usage) {
+  const p = pricingFor(model);
+  if (!p) return 0;
+  return usage.inputTokens / 1e6 * p.input + usage.outputTokens / 1e6 * p.output;
+}
+
+// src/tracks.ts
+import { parsePdf } from "@struktur/sdk";
+var trackToPdfOptions = (track) => {
+  switch (track) {
+    case "text":
+      return { includeImages: false, screenshots: false };
+    case "text+embedded":
+      return { includeImages: true, screenshots: false };
+    case "text+screenshots":
+      return { includeImages: false, screenshots: true };
+    case "text+embedded+screenshots":
+      return { includeImages: true, screenshots: true };
+  }
+};
+async function materializeTrack(artifacts, track) {
+  return Promise.all(
+    artifacts.map(async (artifact) => {
+      if (artifact.type !== "pdf") return artifact;
+      const buffer = await artifact.raw();
+      return parsePdf(buffer, trackToPdfOptions(track));
+    })
+  );
+}
+
+// src/scoring/normalize.ts
+var EMPTY = /* @__PURE__ */ Symbol("struktur.benchmarks.empty");
+var stripDiacritics = (s) => s.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+var normalizeString = (s) => stripDiacritics(s.toLowerCase()).replace(/\s+/g, " ").trim();
+var normalizeAggressive = (s) => stripDiacritics(s.toLowerCase()).replace(/[^a-z0-9]+/g, "");
+function normalizeValue(value, metric) {
+  if (value === null || value === void 0) return EMPTY;
+  if (metric === "exact") {
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+      return value;
+    }
+    return JSON.stringify(value);
+  }
+  if (metric === "semantic") {
+    if (typeof value === "number" || typeof value === "boolean") return value;
+    return normalizeAggressive(typeof value === "string" ? value : JSON.stringify(value));
+  }
+  if (typeof value === "number" || typeof value === "boolean") return value;
+  return normalizeString(typeof value === "string" ? value : JSON.stringify(value));
+}
+var RELATIVE_TOLERANCE = 1e-6;
+function valuesEqual(a, b, metric) {
+  const na = normalizeValue(a, metric);
+  const nb = normalizeValue(b, metric);
+  if (na === EMPTY || nb === EMPTY) return na === EMPTY && nb === EMPTY;
+  if (metric === "tolerance" && typeof na === "number" && typeof nb === "number") {
+    const scale = Math.max(1, Math.abs(na), Math.abs(nb));
+    return Math.abs(na - nb) <= RELATIVE_TOLERANCE * scale;
+  }
+  return na === nb;
+}
+
+// src/scoring/score.ts
+var STOPWORDS = /* @__PURE__ */ new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "of",
+  "to",
+  "in",
+  "on",
+  "at",
+  "is",
+  "are",
+  "it",
+  "this",
+  "that",
+  "with",
+  "for",
+  "as",
+  "by",
+  "from",
+  "be",
+  "was",
+  "were",
+  "und",
+  "der",
+  "die",
+  "das",
+  "ein",
+  "eine",
+  "einen",
+  "dem",
+  "den",
+  "mit",
+  "von",
+  "zur",
+  "zum",
+  "auf",
+  "im",
+  "in",
+  "am",
+  "ist",
+  "sind",
+  "werden",
+  "wird",
+  "des",
+  "sich",
+  "auch",
+  "nicht",
+  "als",
+  "bei",
+  "f\xFCr",
+  "\xFCber",
+  "aus",
+  "nach",
+  "einer",
+  "einem",
+  "eine",
+  "zu",
+  "unter",
+  "an",
+  "sowie",
+  "durch",
+  "alle",
+  "dass",
+  "dieser",
+  "diese",
+  "hier",
+  "sein",
+  "ihr",
+  "wir",
+  "man"
+]);
+var tokenizeProse = (s) => {
+  if (typeof s !== "string") return [];
+  return Array.from(
+    new Set(
+      s.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w.length >= 3 && !STOPWORDS.has(w))
+    )
+  );
+};
+function scoreProse(gold, pred) {
+  const s = empty();
+  const gt = tokenizeProse(String(gold ?? ""));
+  const pt = tokenizeProse(String(pred ?? ""));
+  const gs = new Set(gt);
+  const ps = new Set(pt);
+  const tp = gt.filter((w) => ps.has(w)).length;
+  const fp = pt.filter((w) => !gs.has(w)).length;
+  const fn = gt.filter((w) => !ps.has(w)).length;
+  s.tp = tp;
+  s.fp = fp;
+  s.fn = fn;
+  s.totalGold = gt.length;
+  s.totalPred = pt.length;
+  return finalize(s);
+}
+var empty = () => ({
+  tp: 0,
+  fp: 0,
+  fn: 0,
+  totalGold: 0,
+  totalPred: 0,
+  precision: 1,
+  recall: 1,
+  f1: 1,
+  exactMatch: true,
+  fieldErrors: []
+});
+var finalize = (s) => {
+  s.precision = s.tp + s.fp === 0 ? 1 : s.tp / (s.tp + s.fp);
+  s.recall = s.tp + s.fn === 0 ? 1 : s.tp / (s.tp + s.fn);
+  s.f1 = s.precision + s.recall === 0 ? 0 : 2 * s.precision * s.recall / (s.precision + s.recall);
+  s.exactMatch = s.fp === 0 && s.fn === 0;
+  return s;
+};
+var merge = (into, child) => {
+  into.tp += child.tp;
+  into.fp += child.fp;
+  into.fn += child.fn;
+  into.totalGold += child.totalGold;
+  into.totalPred += child.totalPred;
+  into.fieldErrors.push(...child.fieldErrors);
+};
+var isLeaf = (v) => v === null || v === void 0 || typeof v === "string" || typeof v === "number" || typeof v === "boolean";
+var isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+var leafCount = (v) => {
+  if (Array.isArray(v)) return v.reduce((n, item) => n + leafCount(item), 0);
+  if (isObject(v)) {
+    return Object.values(v).reduce((n, item) => n + leafCount(item), 0);
+  }
+  return 1;
+};
+var canonPath = (path4) => path4.replace(/\[[^\]]*\]/g, "[]");
+var metricFor = (spec, path4) => (
+  // field overrides are keyed by the leaf field name (e.g. "area"), robust to
+  // concrete array keys in the path.
+  spec.fields?.[path4.split(".").pop()] ?? spec.default ?? "normalized"
+);
+var alignmentFor = (spec, path4) => spec.arrays?.find((a) => canonPath(a.path) === canonPath(path4)) ?? spec.arrays?.find((a) => canonPath(path4).endsWith(canonPath(a.path)));
+var isSetPath = (spec, path4) => (spec.sets ?? []).some(
+  (s) => canonPath(s) === canonPath(path4) || canonPath(path4).endsWith(canonPath(s))
+);
+function groupByKey(items, key) {
+  const map = /* @__PURE__ */ new Map();
+  const keys = Array.isArray(key) ? key : [key];
+  for (const item of items) {
+    const k = keys.map((kf) => isObject(item) ? item[kf] : void 0).map((v) => String(normalizeValue(v, "normalized"))).join("|");
+    const arr = map.get(k) ?? [];
+    arr.push(item);
+    map.set(k, arr);
+  }
+  return map;
+}
+function scoreLeaf(path4, gold, pred, spec) {
+  const metric = metricFor(spec, path4);
+  if (metric === "prose" && (typeof gold === "string" || typeof pred === "string")) {
+    return scoreProse(gold, pred);
+  }
+  const s = empty();
+  s.totalGold = isLeaf(gold) ? 1 : leafCount(gold);
+  s.totalPred = isLeaf(pred) ? 1 : leafCount(pred);
+  if (valuesEqual(gold, pred, metric)) {
+    s.tp = isLeaf(gold) ? 1 : leafCount(gold);
+    return finalize(s);
+  }
+  const goldEmpty = gold === null || gold === void 0;
+  const predEmpty = pred === null || pred === void 0;
+  if (goldEmpty) s.fp = s.totalPred;
+  else if (predEmpty) s.fn = s.totalGold;
+  else {
+    s.fn = s.totalGold;
+    s.fp = s.totalPred;
+  }
+  s.fieldErrors.push({ path: path4, gold, pred, metric });
+  return finalize(s);
+}
+function scoreIndexed(path4, g, p, spec) {
+  const s = empty();
+  const len = Math.max(g.length, p.length);
+  for (let i = 0; i < len; i++) {
+    merge(s, scoreAt(`${path4}[${i}]`, g[i] ?? null, p[i] ?? null, spec));
+  }
+  return finalize(s);
+}
+function scoreAligned(path4, g, p, key, spec) {
+  const s = empty();
+  const gm = groupByKey(g, key);
+  const pm = groupByKey(p, key);
+  let matchedAny = false;
+  for (const [k, gItems] of gm) {
+    const pItems = pm.get(k);
+    if (!pItems || pItems.length === 0) {
+      for (const gi of gItems) {
+        s.fn += leafCount(gi);
+        s.totalGold += leafCount(gi);
+      }
+      continue;
+    }
+    matchedAny = true;
+    const pairs = Math.min(gItems.length, pItems.length);
+    for (let i = 0; i < pairs; i++) {
+      merge(s, scoreAt(`${path4}[${String(k)}]`, gItems[i], pItems[i], spec));
+    }
+    for (let i = pairs; i < gItems.length; i++) {
+      s.fn += leafCount(gItems[i]);
+      s.totalGold += leafCount(gItems[i]);
+    }
+    for (let i = pairs; i < pItems.length; i++) {
+      s.fp += leafCount(pItems[i]);
+      s.totalPred += leafCount(pItems[i]);
+    }
+  }
+  for (const [k, pItems] of pm) {
+    if (gm.has(k)) continue;
+    for (const pi of pItems) {
+      s.fp += leafCount(pi);
+      s.totalPred += leafCount(pi);
+    }
+  }
+  if (!matchedAny && g.length > 0 && p.length > 0) {
+    return scoreIndexed(path4, g, p, spec);
+  }
+  return finalize(s);
+}
+function scoreSet(path4, g, p, spec) {
+  const s = empty();
+  const goldKeyed = /* @__PURE__ */ new Map();
+  const predKeyed = /* @__PURE__ */ new Map();
+  const metric = metricFor(spec, canonPath(path4));
+  const bump = (map, v) => {
+    const k = String(normalizeValue(v, metric));
+    map.set(k, (map.get(k) ?? 0) + 1);
+  };
+  for (const v of g) bump(goldKeyed, v);
+  for (const v of p) bump(predKeyed, v);
+  s.totalGold = g.length;
+  s.totalPred = p.length;
+  for (const [k, gc] of goldKeyed) {
+    const pc = predKeyed.get(k) ?? 0;
+    const common = Math.min(gc, pc);
+    s.tp += common;
+    s.fn += gc - common;
+    s.fp += pc - common;
+    if (common < gc) {
+      s.fieldErrors.push({
+        path: path4,
+        gold: g.find((v) => String(normalizeValue(v, metric)) === k),
+        pred: null,
+        metric
+      });
+    }
+  }
+  for (const [k, pc] of predKeyed) {
+    const gc = goldKeyed.get(k) ?? 0;
+    if (pc > gc) {
+      s.fieldErrors.push({
+        path: path4,
+        gold: null,
+        pred: p.find((v) => String(normalizeValue(v, metric)) === k),
+        metric
+      });
+    }
+  }
+  return finalize(s);
+}
+function scoreAt(path4, gold, pred, spec) {
+  if (isLeaf(gold) || isLeaf(pred)) {
+    return scoreLeaf(path4, gold, pred, spec);
+  }
+  if (Array.isArray(gold) || Array.isArray(pred)) {
+    const g = Array.isArray(gold) ? gold : [gold];
+    const p = Array.isArray(pred) ? pred : [pred];
+    if (isSetPath(spec, path4)) {
+      return scoreSet(path4, g, p, spec);
+    }
+    const align = alignmentFor(spec, path4);
+    return align ? scoreAligned(path4, g, p, align.key, spec) : scoreIndexed(path4, g, p, spec);
+  }
+  const s = empty();
+  const keys = /* @__PURE__ */ new Set([...Object.keys(gold), ...Object.keys(pred)]);
+  for (const key of keys) {
+    const childPath = path4 === "" ? key : `${path4}.${key}`;
+    const gv = gold[key] ?? null;
+    const pv = pred[key] ?? null;
+    merge(s, scoreAt(childPath, gv, pv, spec));
+  }
+  return finalize(s);
+}
+function scoreData(gold, pred, spec = {}) {
+  return scoreAt("", gold, pred, spec);
+}
+
+// src/runner.ts
+import { extract, resolveModel, toJsonSchema } from "@struktur/sdk";
+import {
+  simple,
+  parallel,
+  sequential,
+  parallelAutoMerge,
+  sequentialAutoMerge,
+  doublePass,
+  doublePassAutoMerge,
+  agent
+} from "@struktur/sdk";
+
+// src/cache.ts
+import { createHash } from "crypto";
+import { mkdir as mkdir2, readFile as readFile2, rename, writeFile as writeFile2 } from "fs/promises";
+import path2 from "path";
+var stableStringify = (v) => {
+  if (v === null) return "null";
+  if (Array.isArray(v)) return `[${v.map(stableStringify).join(",")}]`;
+  if (typeof v === "object") {
+    const rec = v;
+    const keys = Object.keys(rec).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(rec[k])}`).join(",")}}`;
+  }
+  return JSON.stringify(v) ?? String(v);
+};
+var hashKey = (input) => createHash("sha256").update(stableStringify(input)).digest("hex").slice(0, 16);
+async function cacheGet(dir, key) {
+  try {
+    const raw = await readFile2(path2.join(dir, `${key}.json`), "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+}
+async function cacheSet(dir, key, value) {
+  const file = path2.join(dir, `${key}.json`);
+  await mkdir2(dir, { recursive: true });
+  const tmp = `${file}.tmp`;
+  await writeFile2(tmp, JSON.stringify(value, null, 2), "utf8");
+  await rename(tmp, file);
+}
+
+// src/runner.ts
+init_config();
+var builtins = {
+  simple: (model, _spec, instructions) => simple({ model, outputInstructions: instructions }),
+  parallel: (model, _spec, instructions) => parallel({ model, mergeModel: model, chunkSize: 1e4, outputInstructions: instructions }),
+  sequential: (model, _spec, instructions) => sequential({ model, chunkSize: 1e4, outputInstructions: instructions }),
+  parallelAutoMerge: (model, _spec, instructions) => parallelAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  sequentialAutoMerge: (model, _spec, instructions) => sequentialAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  doublePass: (model, _spec, instructions) => doublePass({ model, mergeModel: model, chunkSize: 1e4, outputInstructions: instructions }),
+  doublePassAutoMerge: (model, _spec, instructions) => doublePassAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  agent: (_model, modelSpec, instructions) => {
+    const [provider, ...rest] = modelSpec.split("/");
+    const modelId = rest.join("/");
+    if (!provider || !modelId)
+      throw new Error(`Agent requires 'provider/model'. Got: ${modelSpec}`);
+    return agent({
+      provider,
+      modelId,
+      maxSteps: 50,
+      maxIterations: 1,
+      vision: true,
+      outputInstructions: instructions
+    });
+  }
+};
+function resolveStrategy(entry, model, modelSpec) {
+  if (typeof entry === "string") {
+    const factory = builtins[entry];
+    if (!factory) {
+      throw new Error(
+        `Unknown builtin strategy: ${entry}. Available: ${Object.keys(builtins).join(", ")}`
+      );
+    }
+    return { name: entry, strategy: factory(model, modelSpec) };
+  }
+  if (typeof entry === "function") {
+    const s = entry(model, modelSpec);
+    return { name: s.name, strategy: s };
+  }
+  if ("strategy" in entry) {
+    const inner = entry.strategy;
+    if (typeof inner === "string") {
+      const factory = builtins[inner];
+      if (!factory) throw new Error(`Unknown builtin strategy: ${inner}`);
+      return {
+        name: entry.label ?? inner,
+        strategy: factory(model, modelSpec, entry.instructions)
+      };
+    }
+    const s = inner(model, modelSpec, entry.instructions);
+    return { name: entry.label ?? s.name, strategy: s };
+  }
+  return {
+    name: entry.name,
+    strategy: entry
+  };
+}
+var emptyUsage = () => ({ inputTokens: 0, outputTokens: 0, totalTokens: 0 });
+async function runBenchmark(options) {
+  const modelSpec = options.model ?? BENCHMARK_MODEL;
+  const model = await resolveModel(modelSpec);
+  const cacheDir = options.cacheDir ?? defaultResultsDir();
+  const useCache = options.cache ?? true;
+  const variant = options.variant ?? "";
+  const strats = options.strategies.map((s) => resolveStrategy(s, model, modelSpec));
+  const cells = [];
+  for (const c of options.cases) {
+    const tracks = c.tracks ?? [...TRACKS];
+    for (const track of tracks) {
+      for (const { name, strategy } of strats) {
+        const cacheKey = hashKey({
+          caseId: c.id,
+          track,
+          strategy: name,
+          model: modelSpec,
+          variant,
+          schema: toJsonSchema(c.schema),
+          gold: c.gold
+        });
+        const cached = useCache ? await cacheGet(cacheDir, cacheKey) : void 0;
+        if (cached) {
+          cached.cached = true;
+          if (cached.costUsd === void 0)
+            cached.costUsd = estimateCostUsd(modelSpec, cached.usage);
+          cells.push(cached);
+          options.onCell?.({ caseId: c.id, track, strategy: name, cached: true });
+          continue;
+        }
+        options.onCell?.({ caseId: c.id, track, strategy: name, cached: false });
+        const artifacts = c.artifactsByTrack?.[track] ?? await materializeTrack(c.artifacts, track);
+        const start = Date.now();
+        let data;
+        let usage = emptyUsage();
+        let valid = true;
+        let error;
+        try {
+          const result = await extract({ artifacts, schema: c.schema, strategy });
+          data = result.data;
+          usage = result.usage;
+          if (result.error) {
+            valid = false;
+            error = result.error.message;
+          }
+        } catch (e) {
+          valid = false;
+          error = e.message;
+          data = null;
+        }
+        const cell = {
+          caseId: c.id,
+          track,
+          strategy: name,
+          model: modelSpec,
+          cached: false,
+          valid,
+          error,
+          score: scoreData(c.gold, c.transform ? c.transform(data) : data, c.metrics),
+          usage,
+          latencyMs: Date.now() - start,
+          costUsd: estimateCostUsd(modelSpec, usage),
+          data
+        };
+        if (useCache) await cacheSet(cacheDir, cacheKey, cell);
+        cells.push(cell);
+      }
+    }
+  }
+  return buildReport(modelSpec, cells, variant);
+}
+function buildReport(model, cells, variant) {
+  const summary = /* @__PURE__ */ new Map();
+  for (const cell of cells) {
+    const key = `${cell.strategy}|${cell.track}`;
+    let row = summary.get(key);
+    if (!row) {
+      row = {
+        strategy: cell.strategy,
+        track: cell.track,
+        cases: 0,
+        meanF1: 0,
+        meanPrecision: 0,
+        meanRecall: 0,
+        validityRate: 0,
+        exactMatchRate: 0,
+        totalInputTokens: 0,
+        totalOutputTokens: 0,
+        totalCostUsd: 0,
+        meanCostUsd: 0,
+        totalLatencyMs: 0,
+        meanLatencyMs: 0
+      };
+      summary.set(key, row);
+    }
+    row.cases++;
+    row.meanF1 += cell.score.f1;
+    row.meanPrecision += cell.score.precision;
+    row.meanRecall += cell.score.recall;
+    if (cell.valid) row.validityRate++;
+    if (cell.score.exactMatch) row.exactMatchRate++;
+    row.totalInputTokens += cell.usage.inputTokens;
+    row.totalOutputTokens += cell.usage.outputTokens;
+    row.totalCostUsd += cell.costUsd ?? estimateCostUsd(model, cell.usage);
+    row.totalLatencyMs += cell.latencyMs;
+    row.meanLatencyMs += cell.latencyMs;
+  }
+  for (const row of summary.values()) {
+    row.meanF1 /= row.cases;
+    row.meanPrecision /= row.cases;
+    row.meanRecall /= row.cases;
+    row.validityRate /= row.cases;
+    row.exactMatchRate /= row.cases;
+    row.meanCostUsd = row.cases ? row.totalCostUsd / row.cases : 0;
+    row.meanLatencyMs = row.cases ? row.meanLatencyMs / row.cases : 0;
+  }
+  return {
+    model,
+    variant,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    cells,
+    summary: [...summary.values()]
+  };
+}
+
+// src/report.ts
+import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
+import path3 from "path";
+var pct = (v) => `${(v * 100).toFixed(1)}%`;
+function toMarkdownTable(report) {
+  const lines = [];
+  lines.push(`# Benchmark report`);
+  lines.push(``);
+  lines.push(`- model: \`${report.model}\``);
+  if (report.variant) lines.push(`- variant: \`${report.variant}\``);
+  lines.push(`- generated: ${report.generatedAt}`);
+  lines.push(``);
+  lines.push(
+    `| strategy | track | cases | F1 | precision | recall | valid | exact | in tok | out tok | cost (USD) | ms/case | total ms |`
+  );
+  lines.push(`|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|`);
+  for (const row of report.summary) {
+    lines.push(
+      `| ${row.strategy} | ${row.track} | ${row.cases} | ${pct(row.meanF1)} | ${pct(row.meanPrecision)} | ${pct(row.meanRecall)} | ${pct(row.validityRate)} | ${pct(row.exactMatchRate)} | ${row.totalInputTokens} | ${row.totalOutputTokens} | ${row.totalCostUsd.toFixed(4)} | ${Math.round(row.meanLatencyMs)} | ${row.totalLatencyMs} |`
+    );
+  }
+  return lines.join("\n");
+}
+async function saveReport(report, file) {
+  await mkdir3(path3.dirname(file), { recursive: true });
+  await writeFile3(file, JSON.stringify(report, null, 2), "utf8");
+  const mdFile = file.replace(/\.json$/, "") + ".md";
+  await writeFile3(mdFile, toMarkdownTable(report), "utf8");
+}
+
+// src/cli.ts
+var args = process.argv.slice(2);
+var cmd = args[0];
+async function main() {
+  switch (cmd) {
+    case "ls": {
+      const { readdir } = await import("fs/promises");
+      const { defaultDatasetDir: defaultDatasetDir2 } = await Promise.resolve().then(() => (init_config(), config_exports));
+      const dir = defaultDatasetDir2();
+      try {
+        const entries = await readdir(dir, { withFileTypes: true });
+        for (const e of entries) {
+          if (e.isDirectory()) console.log(`  ${e.name}`);
+        }
+      } catch {
+        console.log(`Cache dir ${dir} is empty or missing.`);
+      }
+      break;
+    }
+    case "fetch": {
+      const all = args.includes("--all");
+      const name = args[1];
+      const datasets = { sroie };
+      if (all) {
+        for (const [key, ds] of Object.entries(datasets)) {
+          console.log(`Fetching ${key}...`);
+          await loadDataset(ds, { force: true });
+          console.log(`  done.`);
+        }
+      } else if (name && name in datasets) {
+        console.log(`Fetching ${name}...`);
+        await loadDataset(datasets[name], { force: true });
+        console.log(`  done.`);
+      } else {
+        console.log("Usage: struktur-benchmarks fetch <dataset> | --all");
+        console.log(`Available: ${Object.keys(datasets).join(", ")}`);
+      }
+      break;
+    }
+    case "run": {
+      const modulePath = args[1];
+      if (!modulePath) {
+        console.log("Usage: struktur-benchmarks run <module.js>");
+        console.log("  Module must export: { cases, strategies }");
+        process.exit(1);
+      }
+      const mod = await import(resolve(modulePath));
+      const report = await runBenchmark({
+        cases: mod.cases,
+        strategies: mod.strategies ?? ["simple"]
+      });
+      console.log(toMarkdownTable(report));
+      await saveReport(report, `benchmark-report-${Date.now()}.json`);
+      break;
+    }
+    default:
+      console.log("Usage: struktur-benchmarks <ls|fetch|run>");
+      console.log("  ls        List cached datasets");
+      console.log("  fetch     Download a dataset");
+      console.log("  run       Run a module");
+      break;
+  }
+}
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map