@struktur/benchmarks 2.6.0

package/dist/index.js

@@ -0,0 +1,686 @@
+// src/types.ts
+var TRACKS = [
+  "text",
+  "text+embedded",
+  "text+screenshots",
+  "text+embedded+screenshots"
+];
+function defineCase(c) {
+  return c;
+}
+
+// src/config.ts
+var BENCHMARK_MODEL = "openrouter/deepseek/deepseek-v4-flash-vision-exp";
+var defaultDatasetDir = () => process.env.STRUKTUR_BENCHMARKS_DIR ?? `${process.env.HOME ?? "."}/.struktur/benchmarks`;
+var defaultResultsDir = () => `${defaultDatasetDir()}/results`;
+
+// src/pricing.ts
+var PRICING = {
+  // OpenRouter: $0.22/M input, $0.66/M output
+  "openrouter/deepseek/deepseek-v4-flash-vision-exp": { input: 0.22, output: 0.66 },
+  // OpenAI via OpenRouter: $0.15/M input, $0.60/M output
+  "openrouter/openai/gpt-4o-mini": { input: 0.15, output: 0.6 }
+};
+var envOverride = (() => {
+  const raw = process.env.STRUKTUR_BENCHMARK_PRICES;
+  if (!raw) return void 0;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+})();
+var normalizeModel = (model) => model.replace(/#.*$/, "");
+function pricingFor(model) {
+  const key = normalizeModel(model);
+  return envOverride?.[key] ?? PRICING[key];
+}
+function estimateCostUsd(model, usage) {
+  const p = pricingFor(model);
+  if (!p) return 0;
+  return usage.inputTokens / 1e6 * p.input + usage.outputTokens / 1e6 * p.output;
+}
+
+// src/tracks.ts
+import { parsePdf } from "@struktur/sdk";
+var trackToPdfOptions = (track) => {
+  switch (track) {
+    case "text":
+      return { includeImages: false, screenshots: false };
+    case "text+embedded":
+      return { includeImages: true, screenshots: false };
+    case "text+screenshots":
+      return { includeImages: false, screenshots: true };
+    case "text+embedded+screenshots":
+      return { includeImages: true, screenshots: true };
+  }
+};
+async function materializeTrack(artifacts, track) {
+  return Promise.all(
+    artifacts.map(async (artifact) => {
+      if (artifact.type !== "pdf") return artifact;
+      const buffer = await artifact.raw();
+      return parsePdf(buffer, trackToPdfOptions(track));
+    })
+  );
+}
+
+// src/scoring/normalize.ts
+var EMPTY = /* @__PURE__ */ Symbol("struktur.benchmarks.empty");
+var stripDiacritics = (s) => s.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+var normalizeString = (s) => stripDiacritics(s.toLowerCase()).replace(/\s+/g, " ").trim();
+var normalizeAggressive = (s) => stripDiacritics(s.toLowerCase()).replace(/[^a-z0-9]+/g, "");
+function normalizeValue(value, metric) {
+  if (value === null || value === void 0) return EMPTY;
+  if (metric === "exact") {
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+      return value;
+    }
+    return JSON.stringify(value);
+  }
+  if (metric === "semantic") {
+    if (typeof value === "number" || typeof value === "boolean") return value;
+    return normalizeAggressive(typeof value === "string" ? value : JSON.stringify(value));
+  }
+  if (typeof value === "number" || typeof value === "boolean") return value;
+  return normalizeString(typeof value === "string" ? value : JSON.stringify(value));
+}
+var RELATIVE_TOLERANCE = 1e-6;
+function valuesEqual(a, b, metric) {
+  const na = normalizeValue(a, metric);
+  const nb = normalizeValue(b, metric);
+  if (na === EMPTY || nb === EMPTY) return na === EMPTY && nb === EMPTY;
+  if (metric === "tolerance" && typeof na === "number" && typeof nb === "number") {
+    const scale = Math.max(1, Math.abs(na), Math.abs(nb));
+    return Math.abs(na - nb) <= RELATIVE_TOLERANCE * scale;
+  }
+  return na === nb;
+}
+
+// src/scoring/score.ts
+var STOPWORDS = /* @__PURE__ */ new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "of",
+  "to",
+  "in",
+  "on",
+  "at",
+  "is",
+  "are",
+  "it",
+  "this",
+  "that",
+  "with",
+  "for",
+  "as",
+  "by",
+  "from",
+  "be",
+  "was",
+  "were",
+  "und",
+  "der",
+  "die",
+  "das",
+  "ein",
+  "eine",
+  "einen",
+  "dem",
+  "den",
+  "mit",
+  "von",
+  "zur",
+  "zum",
+  "auf",
+  "im",
+  "in",
+  "am",
+  "ist",
+  "sind",
+  "werden",
+  "wird",
+  "des",
+  "sich",
+  "auch",
+  "nicht",
+  "als",
+  "bei",
+  "f\xFCr",
+  "\xFCber",
+  "aus",
+  "nach",
+  "einer",
+  "einem",
+  "eine",
+  "zu",
+  "unter",
+  "an",
+  "sowie",
+  "durch",
+  "alle",
+  "dass",
+  "dieser",
+  "diese",
+  "hier",
+  "sein",
+  "ihr",
+  "wir",
+  "man"
+]);
+var tokenizeProse = (s) => {
+  if (typeof s !== "string") return [];
+  return Array.from(
+    new Set(
+      s.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w.length >= 3 && !STOPWORDS.has(w))
+    )
+  );
+};
+function scoreProse(gold, pred) {
+  const s = empty();
+  const gt = tokenizeProse(String(gold ?? ""));
+  const pt = tokenizeProse(String(pred ?? ""));
+  const gs = new Set(gt);
+  const ps = new Set(pt);
+  const tp = gt.filter((w) => ps.has(w)).length;
+  const fp = pt.filter((w) => !gs.has(w)).length;
+  const fn = gt.filter((w) => !ps.has(w)).length;
+  s.tp = tp;
+  s.fp = fp;
+  s.fn = fn;
+  s.totalGold = gt.length;
+  s.totalPred = pt.length;
+  return finalize(s);
+}
+var empty = () => ({
+  tp: 0,
+  fp: 0,
+  fn: 0,
+  totalGold: 0,
+  totalPred: 0,
+  precision: 1,
+  recall: 1,
+  f1: 1,
+  exactMatch: true,
+  fieldErrors: []
+});
+var finalize = (s) => {
+  s.precision = s.tp + s.fp === 0 ? 1 : s.tp / (s.tp + s.fp);
+  s.recall = s.tp + s.fn === 0 ? 1 : s.tp / (s.tp + s.fn);
+  s.f1 = s.precision + s.recall === 0 ? 0 : 2 * s.precision * s.recall / (s.precision + s.recall);
+  s.exactMatch = s.fp === 0 && s.fn === 0;
+  return s;
+};
+var merge = (into, child) => {
+  into.tp += child.tp;
+  into.fp += child.fp;
+  into.fn += child.fn;
+  into.totalGold += child.totalGold;
+  into.totalPred += child.totalPred;
+  into.fieldErrors.push(...child.fieldErrors);
+};
+var isLeaf = (v) => v === null || v === void 0 || typeof v === "string" || typeof v === "number" || typeof v === "boolean";
+var isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+var leafCount = (v) => {
+  if (Array.isArray(v)) return v.reduce((n, item) => n + leafCount(item), 0);
+  if (isObject(v)) {
+    return Object.values(v).reduce((n, item) => n + leafCount(item), 0);
+  }
+  return 1;
+};
+var canonPath = (path3) => path3.replace(/\[[^\]]*\]/g, "[]");
+var metricFor = (spec, path3) => (
+  // field overrides are keyed by the leaf field name (e.g. "area"), robust to
+  // concrete array keys in the path.
+  spec.fields?.[path3.split(".").pop()] ?? spec.default ?? "normalized"
+);
+var alignmentFor = (spec, path3) => spec.arrays?.find((a) => canonPath(a.path) === canonPath(path3)) ?? spec.arrays?.find((a) => canonPath(path3).endsWith(canonPath(a.path)));
+var isSetPath = (spec, path3) => (spec.sets ?? []).some(
+  (s) => canonPath(s) === canonPath(path3) || canonPath(path3).endsWith(canonPath(s))
+);
+function groupByKey(items, key) {
+  const map = /* @__PURE__ */ new Map();
+  const keys = Array.isArray(key) ? key : [key];
+  for (const item of items) {
+    const k = keys.map((kf) => isObject(item) ? item[kf] : void 0).map((v) => String(normalizeValue(v, "normalized"))).join("|");
+    const arr = map.get(k) ?? [];
+    arr.push(item);
+    map.set(k, arr);
+  }
+  return map;
+}
+function scoreLeaf(path3, gold, pred, spec) {
+  const metric = metricFor(spec, path3);
+  if (metric === "prose" && (typeof gold === "string" || typeof pred === "string")) {
+    return scoreProse(gold, pred);
+  }
+  const s = empty();
+  s.totalGold = isLeaf(gold) ? 1 : leafCount(gold);
+  s.totalPred = isLeaf(pred) ? 1 : leafCount(pred);
+  if (valuesEqual(gold, pred, metric)) {
+    s.tp = isLeaf(gold) ? 1 : leafCount(gold);
+    return finalize(s);
+  }
+  const goldEmpty = gold === null || gold === void 0;
+  const predEmpty = pred === null || pred === void 0;
+  if (goldEmpty) s.fp = s.totalPred;
+  else if (predEmpty) s.fn = s.totalGold;
+  else {
+    s.fn = s.totalGold;
+    s.fp = s.totalPred;
+  }
+  s.fieldErrors.push({ path: path3, gold, pred, metric });
+  return finalize(s);
+}
+function scoreIndexed(path3, g, p, spec) {
+  const s = empty();
+  const len = Math.max(g.length, p.length);
+  for (let i = 0; i < len; i++) {
+    merge(s, scoreAt(`${path3}[${i}]`, g[i] ?? null, p[i] ?? null, spec));
+  }
+  return finalize(s);
+}
+function scoreAligned(path3, g, p, key, spec) {
+  const s = empty();
+  const gm = groupByKey(g, key);
+  const pm = groupByKey(p, key);
+  let matchedAny = false;
+  for (const [k, gItems] of gm) {
+    const pItems = pm.get(k);
+    if (!pItems || pItems.length === 0) {
+      for (const gi of gItems) {
+        s.fn += leafCount(gi);
+        s.totalGold += leafCount(gi);
+      }
+      continue;
+    }
+    matchedAny = true;
+    const pairs = Math.min(gItems.length, pItems.length);
+    for (let i = 0; i < pairs; i++) {
+      merge(s, scoreAt(`${path3}[${String(k)}]`, gItems[i], pItems[i], spec));
+    }
+    for (let i = pairs; i < gItems.length; i++) {
+      s.fn += leafCount(gItems[i]);
+      s.totalGold += leafCount(gItems[i]);
+    }
+    for (let i = pairs; i < pItems.length; i++) {
+      s.fp += leafCount(pItems[i]);
+      s.totalPred += leafCount(pItems[i]);
+    }
+  }
+  for (const [k, pItems] of pm) {
+    if (gm.has(k)) continue;
+    for (const pi of pItems) {
+      s.fp += leafCount(pi);
+      s.totalPred += leafCount(pi);
+    }
+  }
+  if (!matchedAny && g.length > 0 && p.length > 0) {
+    return scoreIndexed(path3, g, p, spec);
+  }
+  return finalize(s);
+}
+function scoreSet(path3, g, p, spec) {
+  const s = empty();
+  const goldKeyed = /* @__PURE__ */ new Map();
+  const predKeyed = /* @__PURE__ */ new Map();
+  const metric = metricFor(spec, canonPath(path3));
+  const bump = (map, v) => {
+    const k = String(normalizeValue(v, metric));
+    map.set(k, (map.get(k) ?? 0) + 1);
+  };
+  for (const v of g) bump(goldKeyed, v);
+  for (const v of p) bump(predKeyed, v);
+  s.totalGold = g.length;
+  s.totalPred = p.length;
+  for (const [k, gc] of goldKeyed) {
+    const pc = predKeyed.get(k) ?? 0;
+    const common = Math.min(gc, pc);
+    s.tp += common;
+    s.fn += gc - common;
+    s.fp += pc - common;
+    if (common < gc) {
+      s.fieldErrors.push({
+        path: path3,
+        gold: g.find((v) => String(normalizeValue(v, metric)) === k),
+        pred: null,
+        metric
+      });
+    }
+  }
+  for (const [k, pc] of predKeyed) {
+    const gc = goldKeyed.get(k) ?? 0;
+    if (pc > gc) {
+      s.fieldErrors.push({
+        path: path3,
+        gold: null,
+        pred: p.find((v) => String(normalizeValue(v, metric)) === k),
+        metric
+      });
+    }
+  }
+  return finalize(s);
+}
+function scoreAt(path3, gold, pred, spec) {
+  if (isLeaf(gold) || isLeaf(pred)) {
+    return scoreLeaf(path3, gold, pred, spec);
+  }
+  if (Array.isArray(gold) || Array.isArray(pred)) {
+    const g = Array.isArray(gold) ? gold : [gold];
+    const p = Array.isArray(pred) ? pred : [pred];
+    if (isSetPath(spec, path3)) {
+      return scoreSet(path3, g, p, spec);
+    }
+    const align = alignmentFor(spec, path3);
+    return align ? scoreAligned(path3, g, p, align.key, spec) : scoreIndexed(path3, g, p, spec);
+  }
+  const s = empty();
+  const keys = /* @__PURE__ */ new Set([...Object.keys(gold), ...Object.keys(pred)]);
+  for (const key of keys) {
+    const childPath = path3 === "" ? key : `${path3}.${key}`;
+    const gv = gold[key] ?? null;
+    const pv = pred[key] ?? null;
+    merge(s, scoreAt(childPath, gv, pv, spec));
+  }
+  return finalize(s);
+}
+function scoreData(gold, pred, spec = {}) {
+  return scoreAt("", gold, pred, spec);
+}
+
+// src/runner.ts
+import { extract, resolveModel, toJsonSchema } from "@struktur/sdk";
+import {
+  simple,
+  parallel,
+  sequential,
+  parallelAutoMerge,
+  sequentialAutoMerge,
+  doublePass,
+  doublePassAutoMerge,
+  agent
+} from "@struktur/sdk";
+
+// src/cache.ts
+import { createHash } from "crypto";
+import { mkdir, readFile, rename, writeFile } from "fs/promises";
+import path from "path";
+var stableStringify = (v) => {
+  if (v === null) return "null";
+  if (Array.isArray(v)) return `[${v.map(stableStringify).join(",")}]`;
+  if (typeof v === "object") {
+    const rec = v;
+    const keys = Object.keys(rec).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(rec[k])}`).join(",")}}`;
+  }
+  return JSON.stringify(v) ?? String(v);
+};
+var hashKey = (input) => createHash("sha256").update(stableStringify(input)).digest("hex").slice(0, 16);
+async function cacheGet(dir, key) {
+  try {
+    const raw = await readFile(path.join(dir, `${key}.json`), "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+}
+async function cacheSet(dir, key, value) {
+  const file = path.join(dir, `${key}.json`);
+  await mkdir(dir, { recursive: true });
+  const tmp = `${file}.tmp`;
+  await writeFile(tmp, JSON.stringify(value, null, 2), "utf8");
+  await rename(tmp, file);
+}
+
+// src/runner.ts
+var builtins = {
+  simple: (model, _spec, instructions) => simple({ model, outputInstructions: instructions }),
+  parallel: (model, _spec, instructions) => parallel({ model, mergeModel: model, chunkSize: 1e4, outputInstructions: instructions }),
+  sequential: (model, _spec, instructions) => sequential({ model, chunkSize: 1e4, outputInstructions: instructions }),
+  parallelAutoMerge: (model, _spec, instructions) => parallelAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  sequentialAutoMerge: (model, _spec, instructions) => sequentialAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  doublePass: (model, _spec, instructions) => doublePass({ model, mergeModel: model, chunkSize: 1e4, outputInstructions: instructions }),
+  doublePassAutoMerge: (model, _spec, instructions) => doublePassAutoMerge({
+    model,
+    dedupeModel: model,
+    chunkSize: 1e4,
+    outputInstructions: instructions
+  }),
+  agent: (_model, modelSpec, instructions) => {
+    const [provider, ...rest] = modelSpec.split("/");
+    const modelId = rest.join("/");
+    if (!provider || !modelId)
+      throw new Error(`Agent requires 'provider/model'. Got: ${modelSpec}`);
+    return agent({
+      provider,
+      modelId,
+      maxSteps: 50,
+      maxIterations: 1,
+      vision: true,
+      outputInstructions: instructions
+    });
+  }
+};
+function resolveStrategy(entry, model, modelSpec) {
+  if (typeof entry === "string") {
+    const factory = builtins[entry];
+    if (!factory) {
+      throw new Error(
+        `Unknown builtin strategy: ${entry}. Available: ${Object.keys(builtins).join(", ")}`
+      );
+    }
+    return { name: entry, strategy: factory(model, modelSpec) };
+  }
+  if (typeof entry === "function") {
+    const s = entry(model, modelSpec);
+    return { name: s.name, strategy: s };
+  }
+  if ("strategy" in entry) {
+    const inner = entry.strategy;
+    if (typeof inner === "string") {
+      const factory = builtins[inner];
+      if (!factory) throw new Error(`Unknown builtin strategy: ${inner}`);
+      return {
+        name: entry.label ?? inner,
+        strategy: factory(model, modelSpec, entry.instructions)
+      };
+    }
+    const s = inner(model, modelSpec, entry.instructions);
+    return { name: entry.label ?? s.name, strategy: s };
+  }
+  return {
+    name: entry.name,
+    strategy: entry
+  };
+}
+var emptyUsage = () => ({ inputTokens: 0, outputTokens: 0, totalTokens: 0 });
+async function runBenchmark(options) {
+  const modelSpec = options.model ?? BENCHMARK_MODEL;
+  const model = await resolveModel(modelSpec);
+  const cacheDir = options.cacheDir ?? defaultResultsDir();
+  const useCache = options.cache ?? true;
+  const variant = options.variant ?? "";
+  const strats = options.strategies.map((s) => resolveStrategy(s, model, modelSpec));
+  const cells = [];
+  for (const c of options.cases) {
+    const tracks = c.tracks ?? [...TRACKS];
+    for (const track of tracks) {
+      for (const { name, strategy } of strats) {
+        const cacheKey = hashKey({
+          caseId: c.id,
+          track,
+          strategy: name,
+          model: modelSpec,
+          variant,
+          schema: toJsonSchema(c.schema),
+          gold: c.gold
+        });
+        const cached = useCache ? await cacheGet(cacheDir, cacheKey) : void 0;
+        if (cached) {
+          cached.cached = true;
+          if (cached.costUsd === void 0)
+            cached.costUsd = estimateCostUsd(modelSpec, cached.usage);
+          cells.push(cached);
+          options.onCell?.({ caseId: c.id, track, strategy: name, cached: true });
+          continue;
+        }
+        options.onCell?.({ caseId: c.id, track, strategy: name, cached: false });
+        const artifacts = c.artifactsByTrack?.[track] ?? await materializeTrack(c.artifacts, track);
+        const start = Date.now();
+        let data;
+        let usage = emptyUsage();
+        let valid = true;
+        let error;
+        try {
+          const result = await extract({ artifacts, schema: c.schema, strategy });
+          data = result.data;
+          usage = result.usage;
+          if (result.error) {
+            valid = false;
+            error = result.error.message;
+          }
+        } catch (e) {
+          valid = false;
+          error = e.message;
+          data = null;
+        }
+        const cell = {
+          caseId: c.id,
+          track,
+          strategy: name,
+          model: modelSpec,
+          cached: false,
+          valid,
+          error,
+          score: scoreData(c.gold, c.transform ? c.transform(data) : data, c.metrics),
+          usage,
+          latencyMs: Date.now() - start,
+          costUsd: estimateCostUsd(modelSpec, usage),
+          data
+        };
+        if (useCache) await cacheSet(cacheDir, cacheKey, cell);
+        cells.push(cell);
+      }
+    }
+  }
+  return buildReport(modelSpec, cells, variant);
+}
+function buildReport(model, cells, variant) {
+  const summary = /* @__PURE__ */ new Map();
+  for (const cell of cells) {
+    const key = `${cell.strategy}|${cell.track}`;
+    let row = summary.get(key);
+    if (!row) {
+      row = {
+        strategy: cell.strategy,
+        track: cell.track,
+        cases: 0,
+        meanF1: 0,
+        meanPrecision: 0,
+        meanRecall: 0,
+        validityRate: 0,
+        exactMatchRate: 0,
+        totalInputTokens: 0,
+        totalOutputTokens: 0,
+        totalCostUsd: 0,
+        meanCostUsd: 0,
+        totalLatencyMs: 0,
+        meanLatencyMs: 0
+      };
+      summary.set(key, row);
+    }
+    row.cases++;
+    row.meanF1 += cell.score.f1;
+    row.meanPrecision += cell.score.precision;
+    row.meanRecall += cell.score.recall;
+    if (cell.valid) row.validityRate++;
+    if (cell.score.exactMatch) row.exactMatchRate++;
+    row.totalInputTokens += cell.usage.inputTokens;
+    row.totalOutputTokens += cell.usage.outputTokens;
+    row.totalCostUsd += cell.costUsd ?? estimateCostUsd(model, cell.usage);
+    row.totalLatencyMs += cell.latencyMs;
+    row.meanLatencyMs += cell.latencyMs;
+  }
+  for (const row of summary.values()) {
+    row.meanF1 /= row.cases;
+    row.meanPrecision /= row.cases;
+    row.meanRecall /= row.cases;
+    row.validityRate /= row.cases;
+    row.exactMatchRate /= row.cases;
+    row.meanCostUsd = row.cases ? row.totalCostUsd / row.cases : 0;
+    row.meanLatencyMs = row.cases ? row.meanLatencyMs / row.cases : 0;
+  }
+  return {
+    model,
+    variant,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    cells,
+    summary: [...summary.values()]
+  };
+}
+
+// src/report.ts
+import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
+import path2 from "path";
+var pct = (v) => `${(v * 100).toFixed(1)}%`;
+function toMarkdownTable(report) {
+  const lines = [];
+  lines.push(`# Benchmark report`);
+  lines.push(``);
+  lines.push(`- model: \`${report.model}\``);
+  if (report.variant) lines.push(`- variant: \`${report.variant}\``);
+  lines.push(`- generated: ${report.generatedAt}`);
+  lines.push(``);
+  lines.push(
+    `| strategy | track | cases | F1 | precision | recall | valid | exact | in tok | out tok | cost (USD) | ms/case | total ms |`
+  );
+  lines.push(`|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|`);
+  for (const row of report.summary) {
+    lines.push(
+      `| ${row.strategy} | ${row.track} | ${row.cases} | ${pct(row.meanF1)} | ${pct(row.meanPrecision)} | ${pct(row.meanRecall)} | ${pct(row.validityRate)} | ${pct(row.exactMatchRate)} | ${row.totalInputTokens} | ${row.totalOutputTokens} | ${row.totalCostUsd.toFixed(4)} | ${Math.round(row.meanLatencyMs)} | ${row.totalLatencyMs} |`
+    );
+  }
+  return lines.join("\n");
+}
+async function saveReport(report, file) {
+  await mkdir2(path2.dirname(file), { recursive: true });
+  await writeFile2(file, JSON.stringify(report, null, 2), "utf8");
+  const mdFile = file.replace(/\.json$/, "") + ".md";
+  await writeFile2(mdFile, toMarkdownTable(report), "utf8");
+}
+export {
+  BENCHMARK_MODEL,
+  EMPTY,
+  TRACKS,
+  buildReport,
+  defaultDatasetDir,
+  defaultResultsDir,
+  defineCase,
+  estimateCostUsd,
+  leafCount,
+  materializeTrack,
+  normalizeAggressive,
+  normalizeString,
+  normalizeValue,
+  pricingFor,
+  runBenchmark,
+  saveReport,
+  scoreData,
+  toMarkdownTable,
+  trackToPdfOptions,
+  valuesEqual
+};
+//# sourceMappingURL=index.js.map