@workglow/eval 0.3.26

package/dist/workglow-eval.js

@@ -0,0 +1,1266 @@
+#!/usr/bin/env bun
+// @bun
+
+// src/workglow-eval.ts
+import { program } from "commander";
+
+// src/hf/auth.ts
+function hfAuthHeaders(token) {
+  const t = token ?? process.env.HF_TOKEN;
+  return t ? { Authorization: `Bearer ${t}` } : {};
+}
+
+// src/hf/ids.ts
+var ID_SEGMENT = /^[\w-][\w.-]*$/;
+function sanitizeHubRepoId(id) {
+  const segments = id.split("/");
+  if (segments.length < 1 || segments.length > 2) {
+    throw new Error(`invalid HuggingFace repo id "${id}" \u2014 expected "name" or "org/name"`);
+  }
+  return segments.map((segment) => encodeIdSegment(segment, id)).join("/");
+}
+function sanitizeHubFilePath(path) {
+  const segments = path.split("/");
+  if (segments.length === 0)
+    throw new Error(`invalid hub file path "${path}"`);
+  return segments.map((segment) => {
+    if (segment === "" || segment === "." || segment === "..") {
+      throw new Error(`invalid hub file path segment in "${path}"`);
+    }
+    return encodeURIComponent(segment);
+  }).join("/");
+}
+function encodeIdSegment(segment, id) {
+  if (!ID_SEGMENT.test(segment)) {
+    throw new Error(`invalid HuggingFace repo id "${id}" \u2014 segments may contain letters, digits, ` + `".", "_", "-" and must not start with "."`);
+  }
+  return encodeURIComponent(segment);
+}
+
+// src/hf/datasetsServer.ts
+var DATASETS_SERVER = "https://datasets-server.huggingface.co";
+var PAGE_SIZE = 100;
+async function getJson(url, token) {
+  const res = await fetch(url, { headers: hfAuthHeaders(token) });
+  if (!res.ok) {
+    throw new Error(`datasets-server request failed (${res.status}): ${url}`);
+  }
+  return await res.json();
+}
+async function resolveConfig(dataset, token) {
+  const data = await getJson(`${DATASETS_SERVER}/splits?dataset=${encodeURIComponent(dataset)}`, token);
+  const config = data.splits[0]?.config;
+  if (!config)
+    throw new Error(`datasets-server reported no configs for ${dataset}`);
+  return config;
+}
+function extractLabelNames(features) {
+  const labelNames = {};
+  for (const feature of features) {
+    const type = feature.type;
+    if (type?._type === "ClassLabel" && Array.isArray(type.names)) {
+      labelNames[feature.name] = type.names;
+    }
+  }
+  return labelNames;
+}
+async function fetchViaDatasetsServer(options) {
+  const { split, limit, token } = options;
+  const dataset = sanitizeHubRepoId(options.dataset);
+  const config = options.config ?? await resolveConfig(dataset, token);
+  const offset = options.offset ?? 0;
+  const rows = [];
+  let columns = [];
+  let labelNames = {};
+  let truncatedRows = 0;
+  while (rows.length < limit) {
+    const length = Math.min(PAGE_SIZE, limit - rows.length);
+    const url = `${DATASETS_SERVER}/rows?dataset=${encodeURIComponent(dataset)}` + `&config=${encodeURIComponent(config)}&split=${encodeURIComponent(split)}` + `&offset=${offset + rows.length}&length=${length}`;
+    const page = await getJson(url, token);
+    if (rows.length === 0) {
+      columns = page.features.map((f) => f.name);
+      labelNames = extractLabelNames(page.features);
+    }
+    for (const r of page.rows) {
+      if (r.truncated_cells && r.truncated_cells.length > 0)
+        truncatedRows++;
+      rows.push(r.row);
+    }
+    if (page.rows.length < length || offset + rows.length >= page.num_rows_total)
+      break;
+  }
+  if (truncatedRows > 0) {
+    console.error(`warning: the datasets viewer truncated large cells in ${truncatedRows} row(s); ` + `evals will run on the truncated text`);
+  }
+  return { rows, columns, labelNames, config, source: "datasets-server" };
+}
+
+// src/hf/hubFiles.ts
+import { parquetMetadataAsync, parquetReadObjects } from "hyparquet";
+import { compressors } from "hyparquet-compressors";
+import { gunzipSync } from "zlib";
+var HUB = "https://huggingface.co";
+var SUPPORTED_EXTENSIONS = [".parquet", ".jsonl.gz", ".jsonl", ".ndjson", ".json.gz", ".json"];
+function fileExtension(path) {
+  const lower = path.toLowerCase();
+  return SUPPORTED_EXTENSIONS.find((ext) => lower.endsWith(ext));
+}
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function scoreFileForSplit(path, split) {
+  if (!fileExtension(path))
+    return;
+  const lower = path.toLowerCase();
+  const name = lower.split("/").pop() ?? lower;
+  const s = split.toLowerCase();
+  if (name === `${s}${fileExtension(name)}`)
+    return 3;
+  if (name.startsWith(`${s}-`) || name.startsWith(`${s}.`) || name.startsWith(`${s}_`))
+    return 2;
+  if (new RegExp(`(^|[/_.-])${escapeRegExp(s)}([/_.-]|$)`).test(lower))
+    return 1;
+  return;
+}
+function selectSplitFiles(files, split) {
+  let best = 0;
+  let selected = [];
+  for (const path of files) {
+    const score = scoreFileForSplit(path, split) ?? 0;
+    if (score > best) {
+      best = score;
+      selected = [path];
+    } else if (score === best && score > 0) {
+      selected.push(path);
+    }
+  }
+  return selected.sort();
+}
+function sanitizeValue(value) {
+  if (typeof value === "bigint") {
+    return value >= BigInt(Number.MIN_SAFE_INTEGER) && value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value.toString();
+  }
+  if (value instanceof Date)
+    return value.toISOString();
+  if (Array.isArray(value))
+    return value.map(sanitizeValue);
+  if (value !== null && typeof value === "object" && value.constructor === Object) {
+    const out = {};
+    for (const [key, inner] of Object.entries(value))
+      out[key] = sanitizeValue(inner);
+    return out;
+  }
+  return value;
+}
+function sanitizeRow(row) {
+  return sanitizeValue(row);
+}
+function parseJsonLines(text) {
+  return text.split(`
+`).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+}
+async function parseDatasetFile(path, bytes, limit) {
+  const ext = fileExtension(path);
+  if (ext === ".parquet") {
+    const metadata = await parquetMetadataAsync(bytes);
+    const labelNames = {};
+    const hfMeta = metadata.key_value_metadata?.find((kv) => kv.key === "huggingface");
+    if (hfMeta?.value) {
+      try {
+        const info = JSON.parse(hfMeta.value);
+        for (const [name, feature] of Object.entries(info.info?.features ?? {})) {
+          if (feature?._type === "ClassLabel" && Array.isArray(feature.names)) {
+            labelNames[name] = feature.names;
+          }
+        }
+      } catch {}
+    }
+    const rows = await parquetReadObjects({
+      file: bytes,
+      compressors,
+      metadata,
+      rowEnd: limit
+    });
+    return { rows: rows.map(sanitizeRow), labelNames };
+  }
+  const text = ext?.endsWith(".gz") ? gunzipSync(Buffer.from(bytes)).toString("utf-8") : new TextDecoder().decode(bytes);
+  if (ext === ".json" || ext === ".json.gz") {
+    const parsed = JSON.parse(text);
+    const rows = Array.isArray(parsed) ? parsed : parsed.rows ?? parsed.data ?? [];
+    return { rows: rows.slice(0, limit), labelNames: {} };
+  }
+  return { rows: parseJsonLines(text).slice(0, limit), labelNames: {} };
+}
+async function listRepoFiles(dataset, token) {
+  const url = `${HUB}/api/datasets/${sanitizeHubRepoId(dataset)}/tree/main?recursive=true`;
+  const res = await fetch(url, { headers: hfAuthHeaders(token) });
+  if (!res.ok) {
+    throw new Error(`hub tree listing failed (${res.status}) for dataset ${dataset}`);
+  }
+  const entries = await res.json();
+  return entries.filter((e) => e.type === "file").map((e) => e.path);
+}
+async function downloadRepoFile(dataset, path, token) {
+  const url = `${HUB}/datasets/${sanitizeHubRepoId(dataset)}/resolve/main/${sanitizeHubFilePath(path)}`;
+  const res = await fetch(url, { headers: hfAuthHeaders(token) });
+  if (!res.ok) {
+    throw new Error(`hub file download failed (${res.status}) for ${dataset}/${path}`);
+  }
+  return await res.arrayBuffer();
+}
+async function fetchViaHubFiles(options) {
+  const { dataset, split, limit, token } = options;
+  const allFiles = await listRepoFiles(dataset, token);
+  const configFiles = options.config ? allFiles.filter((f) => f.startsWith(`${options.config}/`) || !f.includes("/")) : allFiles;
+  const files = selectSplitFiles(configFiles, split);
+  if (files.length === 0) {
+    throw new Error(`no data files matching split "${split}" found in ${dataset} ` + `(supported: ${SUPPORTED_EXTENSIONS.join(", ")})`);
+  }
+  const offset = options.offset ?? 0;
+  const wanted = offset + limit;
+  const rows = [];
+  let labelNames = {};
+  for (const file of files) {
+    if (rows.length >= wanted)
+      break;
+    const bytes = await downloadRepoFile(dataset, file, token);
+    const parsed = await parseDatasetFile(file, bytes, wanted - rows.length);
+    rows.push(...parsed.rows);
+    if (Object.keys(parsed.labelNames).length > 0)
+      labelNames = parsed.labelNames;
+  }
+  const windowed = rows.slice(offset, wanted);
+  const columns = [...new Set(windowed.flatMap((row) => Object.keys(row)))];
+  return { rows: windowed, columns, labelNames, config: options.config ?? "", source: "hub-files" };
+}
+
+// src/hf/pullDataset.ts
+async function fetchDataset(options) {
+  try {
+    return await fetchViaDatasetsServer(options);
+  } catch (err) {
+    console.error(`datasets-server unavailable (${err instanceof Error ? err.message : String(err)}); ` + `falling back to direct hub file download`);
+    return await fetchViaHubFiles(options);
+  }
+}
+async function pullDatasetIntoStorage(stores, options) {
+  const fetched = await fetchDataset(options);
+  const { dataset, split } = options;
+  const offset = options.offset ?? 0;
+  await stores.rows.withTransaction(async (tx) => {
+    if (offset > 0) {
+      await tx.deleteSearch({ dataset, split, row_index: { value: offset, operator: ">=" } });
+    } else {
+      await tx.deleteSearch({ dataset, split });
+    }
+    await tx.putBulk(fetched.rows.map((row, i) => ({
+      dataset,
+      split,
+      row_index: offset + i,
+      data: JSON.stringify(row)
+    })));
+  });
+  const numRows = await stores.rows.count({ dataset, split });
+  await stores.datasets.put({
+    dataset,
+    split,
+    config: fetched.config || (options.config ?? ""),
+    num_rows: numRows,
+    columns: JSON.stringify(fetched.columns),
+    label_names: Object.keys(fetched.labelNames).length > 0 ? JSON.stringify(fetched.labelNames) : null,
+    source: fetched.source,
+    fetched_at: new Date().toISOString()
+  });
+  return { numRows, source: fetched.source };
+}
+
+// src/util.ts
+function formatTable(rows, columns) {
+  if (rows.length === 0)
+    return "(none)";
+  const widths = columns.map((col) => Math.max(col.length, ...rows.map((row) => (row[col] ?? "").length)));
+  const header = columns.map((col, i) => col.padEnd(widths[i])).join("  ");
+  const separator = widths.map((w) => "-".repeat(w)).join("  ");
+  const body = rows.map((row) => columns.map((col, i) => (row[col] ?? "").padEnd(widths[i])).join("  ")).join(`
+`);
+  return `${header}
+${separator}
+${body}`;
+}
+function formatMetric(value, digits = 3) {
+  return Number.isNaN(value) ? "-" : value.toFixed(digits);
+}
+function formatError(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function parseIntFlag(value, flag, minimum) {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < minimum) {
+    throw new Error(`${flag} must be an integer >= ${minimum} (got "${value}")`);
+  }
+  return parsed;
+}
+
+// src/commands/dataset.ts
+function registerDatasetCommand(program, openStores) {
+  const dataset = program.command("dataset").description("Pull and inspect HuggingFace datasets stored locally");
+  dataset.command("pull").argument("<id>", "HuggingFace dataset id (e.g. dair-ai/emotion)").option("--split <split>", "dataset split", "test").option("--config <config>", "dataset config name (defaults to the first config)").option("--limit <n>", "maximum rows to pull", "100").option("--offset <n>", "row offset to start from", "0").description("Fetch dataset rows and store them in the local eval database").action(async (id, opts) => {
+    const stores = await openStores();
+    try {
+      const { numRows, source } = await pullDatasetIntoStorage(stores, {
+        dataset: id,
+        split: opts.split,
+        config: opts.config,
+        limit: parseIntFlag(opts.limit, "--limit", 1),
+        offset: parseIntFlag(opts.offset, "--offset", 0)
+      });
+      console.log(`stored ${numRows} rows of ${id} [${opts.split}] (via ${source})`);
+    } catch (err) {
+      console.error(`Error: ${formatError(err)}`);
+      process.exitCode = 1;
+    }
+  });
+  dataset.command("list").description("List locally stored dataset splits").action(async () => {
+    const stores = await openStores();
+    const all = await stores.datasets.getAll() ?? [];
+    const rows = all.map((d) => ({
+      dataset: d.dataset,
+      split: d.split,
+      rows: String(d.num_rows),
+      columns: JSON.parse(d.columns).join(","),
+      source: d.source,
+      fetched: d.fetched_at
+    }));
+    console.log(formatTable(rows, ["dataset", "split", "rows", "columns", "source", "fetched"]));
+  });
+  dataset.command("show").argument("<id>", "HuggingFace dataset id").option("--split <split>", "dataset split", "test").option("--limit <n>", "rows to print", "5").description("Print stored rows of a dataset split as JSON lines").action(async (id, opts) => {
+    const stores = await openStores();
+    try {
+      const limit = parseIntFlag(opts.limit, "--limit", 1);
+      const rows = await stores.rows.query({ dataset: id, split: opts.split });
+      if (!rows || rows.length === 0) {
+        console.error(`No stored rows for ${id} [${opts.split}] \u2014 run \`dataset pull\` first.`);
+        process.exitCode = 1;
+        return;
+      }
+      rows.sort((a, b) => a.row_index - b.row_index);
+      for (const row of rows.slice(0, limit)) {
+        console.log(row.data);
+      }
+    } catch (err) {
+      console.error(`Error: ${formatError(err)}`);
+      process.exitCode = 1;
+    }
+  });
+}
+
+// src/score/classification.ts
+function normalizeLabel(label) {
+  return label.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+function scoreClassification(pairs) {
+  const perLabel = new Map;
+  let correct = 0;
+  for (const { expected, predicted } of pairs) {
+    const key = normalizeLabel(expected);
+    const entry = perLabel.get(key) ?? { total: 0, correct: 0 };
+    entry.total++;
+    if (key === normalizeLabel(predicted)) {
+      entry.correct++;
+      correct++;
+    }
+    perLabel.set(key, entry);
+  }
+  return {
+    scored: pairs.length,
+    correct,
+    accuracy: pairs.length > 0 ? correct / pairs.length : NaN,
+    perLabel
+  };
+}
+
+// src/score/correlation.ts
+function pearson(xs, ys) {
+  const n = Math.min(xs.length, ys.length);
+  if (n < 2)
+    return NaN;
+  let sumX = 0;
+  let sumY = 0;
+  for (let i = 0;i < n; i++) {
+    sumX += xs[i];
+    sumY += ys[i];
+  }
+  const meanX = sumX / n;
+  const meanY = sumY / n;
+  let cov = 0;
+  let varX = 0;
+  let varY = 0;
+  for (let i = 0;i < n; i++) {
+    const dx = xs[i] - meanX;
+    const dy = ys[i] - meanY;
+    cov += dx * dy;
+    varX += dx * dx;
+    varY += dy * dy;
+  }
+  if (varX === 0 || varY === 0)
+    return NaN;
+  return cov / Math.sqrt(varX * varY);
+}
+function ranks(values) {
+  const indexed = values.map((value, index) => ({ value, index }));
+  indexed.sort((a, b) => a.value - b.value);
+  const out = new Array(values.length);
+  let i = 0;
+  while (i < indexed.length) {
+    let j = i;
+    while (j + 1 < indexed.length && indexed[j + 1].value === indexed[i].value)
+      j++;
+    const rank = (i + j) / 2 + 1;
+    for (let k = i;k <= j; k++)
+      out[indexed[k].index] = rank;
+    i = j + 1;
+  }
+  return out;
+}
+function spearman(xs, ys) {
+  const n = Math.min(xs.length, ys.length);
+  return pearson(ranks(xs.slice(0, n)), ranks(ys.slice(0, n)));
+}
+
+// src/score/extraction.ts
+function normalizeValue(value) {
+  if (typeof value === "object" && value !== null)
+    return normalizeLabel(JSON.stringify(value));
+  return normalizeLabel(String(value ?? ""));
+}
+function distinctByKey(rows, keyField) {
+  const map = new Map;
+  for (const row of rows) {
+    const key = normalizeValue(row[keyField]);
+    if (key.length > 0)
+      map.set(key, row);
+  }
+  return map;
+}
+function scoreExtraction(candidate, expected, keyField, fields) {
+  const candidateByKey = distinctByKey(candidate, keyField);
+  const expectedByKey = distinctByKey(expected, keyField);
+  let matched = 0;
+  let candidateMatched = 0;
+  let fieldsTotal = 0;
+  let fieldsAgreed = 0;
+  for (const [key, expectedRow] of expectedByKey) {
+    const candidateRow = candidateByKey.get(key);
+    if (!candidateRow)
+      continue;
+    matched++;
+    const scoredFields = (fields ?? Object.keys(expectedRow).filter((f) => normalizeValue(expectedRow[f]).length > 0)).filter((field) => field !== keyField);
+    for (const field of scoredFields) {
+      const expectedValue = normalizeValue(expectedRow[field]);
+      if (expectedValue.length === 0)
+        continue;
+      fieldsTotal++;
+      if (normalizeValue(candidateRow[field]) === expectedValue)
+        fieldsAgreed++;
+    }
+  }
+  for (const key of candidateByKey.keys()) {
+    if (expectedByKey.has(key))
+      candidateMatched++;
+  }
+  return withRates({
+    expected: expectedByKey.size,
+    matched,
+    candidateDistinct: candidateByKey.size,
+    candidateMatched,
+    fieldsTotal,
+    fieldsAgreed
+  });
+}
+function combineExtractionCounts(counts) {
+  const total = counts.reduce((sum, c) => ({
+    expected: sum.expected + c.expected,
+    matched: sum.matched + c.matched,
+    candidateDistinct: sum.candidateDistinct + c.candidateDistinct,
+    candidateMatched: sum.candidateMatched + c.candidateMatched,
+    fieldsTotal: sum.fieldsTotal + c.fieldsTotal,
+    fieldsAgreed: sum.fieldsAgreed + c.fieldsAgreed
+  }), {
+    expected: 0,
+    matched: 0,
+    candidateDistinct: 0,
+    candidateMatched: 0,
+    fieldsTotal: 0,
+    fieldsAgreed: 0
+  });
+  return withRates(total);
+}
+function withRates(counts) {
+  return {
+    ...counts,
+    score: counts.fieldsTotal > 0 ? counts.fieldsAgreed / counts.fieldsTotal : NaN,
+    found: counts.expected > 0 ? counts.matched / counts.expected : NaN,
+    prec: counts.candidateDistinct > 0 ? counts.candidateMatched / counts.candidateDistinct : NaN
+  };
+}
+
+// src/report/aggregate.ts
+var RANKING_METRIC = {
+  classify: (r) => r.accuracy,
+  similarity: (r) => r.spearman,
+  extract: (r) => Number.isFinite(r.score) ? r.score : r.found
+};
+function aggregateResults(kind, results, options = {}) {
+  const byModel = new Map;
+  for (const result of results) {
+    const list = byModel.get(result.model) ?? [];
+    list.push(result);
+    byModel.set(result.model, list);
+  }
+  const reports = [];
+  for (const [model, rows] of byModel) {
+    const ok = rows.filter((r) => r.ok === 1);
+    const pairs = ok.filter((r) => r.expected != null && r.predicted != null).map((r) => ({ expected: r.expected, predicted: r.predicted }));
+    const numeric = kind === "similarity" ? ok.filter((r) => Number.isFinite(r.expected_value ?? NaN) && Number.isFinite(r.predicted_value ?? NaN)) : [];
+    const expectedValues = numeric.map((r) => r.expected_value);
+    const predictedValues = numeric.map((r) => r.predicted_value);
+    const extraction = kind === "extract" ? combineExtractionCounts(extractionCountsFor(pairs, options)) : { score: NaN, found: NaN, prec: NaN };
+    const latency = rows.length > 0 ? rows.reduce((sum, r) => sum + r.latency_ms, 0) / rows.length : NaN;
+    reports.push({
+      model,
+      rows: rows.length,
+      okRows: ok.length,
+      accuracy: kind === "classify" ? scoreClassification(pairs).accuracy : NaN,
+      pearson: pearson(predictedValues, expectedValues),
+      spearman: spearman(predictedValues, expectedValues),
+      score: extraction.score,
+      found: extraction.found,
+      prec: extraction.prec,
+      avgLatencyMs: latency
+    });
+  }
+  const metric = (r) => {
+    const value = RANKING_METRIC[kind](r);
+    return Number.isNaN(value) ? -Infinity : value;
+  };
+  reports.sort((a, b) => metric(b) - metric(a) || a.avgLatencyMs - b.avgLatencyMs);
+  return reports;
+}
+function extractionCountsFor(pairs, options) {
+  const keyField = options.keyField ?? "name";
+  const counts = [];
+  for (const pair of pairs) {
+    try {
+      const expected = JSON.parse(pair.expected);
+      const predicted = JSON.parse(pair.predicted);
+      if (!Array.isArray(expected) || !Array.isArray(predicted))
+        continue;
+      counts.push(scoreExtraction(predicted, expected, keyField, options.fields));
+    } catch {
+      continue;
+    }
+  }
+  return counts;
+}
+
+// src/commands/report.ts
+async function printReport(stores, runId, format) {
+  const run = await stores.runs.get({ run_id: runId });
+  if (!run)
+    throw new Error(`run ${runId} not found`);
+  if (run.kind !== "classify" && run.kind !== "similarity" && run.kind !== "extract") {
+    throw new Error(`run ${runId} has unknown eval kind "${run.kind}"`);
+  }
+  const kind = run.kind;
+  const results = await stores.results.query({ run_id: runId }) ?? [];
+  let runOptions = {};
+  if (kind === "extract") {
+    try {
+      const parsed = JSON.parse(run.options);
+      if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+        runOptions = parsed;
+      }
+    } catch {}
+  }
+  const reports = aggregateResults(kind, results, runOptions);
+  if (format === "json") {
+    console.log(JSON.stringify({ run, reports }, null, 2));
+    return;
+  }
+  console.log(`run ${run.run_id} \u2014 ${run.kind} on ${run.dataset} [${run.split}]`);
+  const REPORT_COLUMNS = {
+    classify: ["model", "rows", "ok", "accuracy", "avg_ms"],
+    similarity: ["model", "rows", "ok", "pearson", "spearman", "avg_ms"],
+    extract: ["model", "rows", "ok", "score", "found", "prec", "avg_ms"]
+  };
+  const tableRows = reports.map((r) => ({
+    model: r.model,
+    rows: String(r.rows),
+    ok: String(r.okRows),
+    accuracy: formatMetric(r.accuracy),
+    pearson: formatMetric(r.pearson),
+    spearman: formatMetric(r.spearman),
+    score: formatMetric(r.score),
+    found: formatMetric(r.found),
+    prec: formatMetric(r.prec),
+    avg_ms: formatMetric(r.avgLatencyMs, 0)
+  }));
+  console.log(formatTable(tableRows, REPORT_COLUMNS[kind]));
+  const failures = results.filter((r) => r.ok !== 1);
+  if (failures.length > 0) {
+    const sample = failures[0];
+    console.log(`
+${failures.length} failed execution(s); first: ` + `${sample.model} row ${sample.row_index}: ${sample.error ?? "unknown error"}`);
+  }
+}
+function registerReportCommand(program, openStores) {
+  program.command("runs").description("List stored eval runs").action(async () => {
+    const stores = await openStores();
+    const all = await stores.runs.getAll() ?? [];
+    all.sort((a, b) => b.created_at.localeCompare(a.created_at));
+    const rows = all.map((r) => ({
+      run: r.run_id,
+      kind: r.kind,
+      dataset: `${r.dataset} [${r.split}]`,
+      models: JSON.parse(r.models).join(","),
+      created: r.created_at
+    }));
+    console.log(formatTable(rows, ["run", "kind", "dataset", "models", "created"]));
+  });
+  program.command("report").argument("[runId]", "run id (defaults to the most recent run)").option("--format <fmt>", "table or json", "table").description("Score a stored run and rank its models").action(async (runId, opts) => {
+    const stores = await openStores();
+    try {
+      let target = runId;
+      if (!target) {
+        const all = await stores.runs.getAll() ?? [];
+        all.sort((a, b) => b.created_at.localeCompare(a.created_at));
+        target = all[0]?.run_id;
+      }
+      if (!target) {
+        throw new Error("no stored runs \u2014 run `run-classify`, `run-similarity`, or `run-extract`");
+      }
+      await printReport(stores, target, opts.format);
+    } catch (err) {
+      console.error(`Error: ${formatError(err)}`);
+      process.exitCode = 1;
+    }
+  });
+}
+
+// src/models.ts
+import { downloadModel } from "@workglow/ai";
+
+// src/config.ts
+import { mkdirSync } from "fs";
+import { homedir } from "os";
+import { join, resolve } from "path";
+function evalHome() {
+  const override = process.env.WORKGLOW_EVAL_HOME;
+  return override ? resolve(override) : join(homedir(), ".workglow", "eval");
+}
+function ggufCacheDir() {
+  return join(evalHome(), "cache", "gguf");
+}
+function loadConfig() {
+  const home = evalHome();
+  mkdirSync(home, { recursive: true });
+  return {
+    home,
+    dbPath: join(home, "eval.sqlite"),
+    modelCache: join(home, "cache", "onnx")
+  };
+}
+
+// src/models.ts
+var GGUF_PREFIX = "gguf:";
+function resolveModelConfig(id, kind) {
+  if (id.startsWith(GGUF_PREFIX)) {
+    const ref = id.slice(GGUF_PREFIX.length);
+    const provider_config = {
+      models_dir: ggufCacheDir()
+    };
+    if (kind === "similarity")
+      provider_config.embedding = true;
+    if (/^(?:hf:|https?:)/.test(ref)) {
+      provider_config.model_url = ref;
+    } else if (ref.endsWith(".gguf")) {
+      provider_config.model_path = ref;
+    } else {
+      provider_config.model_url = `hf:${ref}`;
+    }
+    return { provider: "LOCAL_LLAMACPP", provider_config };
+  }
+  if (id.includes("/")) {
+    const [path, dtype] = splitDtype(id);
+    return {
+      provider: "HF_TRANSFORMERS_ONNX",
+      provider_config: {
+        model_path: path,
+        pipeline: kind === "similarity" ? "feature-extraction" : "text-generation",
+        dtype: dtype ?? (kind === "similarity" ? "q8" : "q4")
+      }
+    };
+  }
+  if (/^claude-/.test(id)) {
+    return { provider: "ANTHROPIC", provider_config: { model_name: id, max_tokens: 1024 } };
+  }
+  if (/^(?:gpt-|o\d|chatgpt-|text-embedding-)/.test(id)) {
+    return { provider: "OPENAI", provider_config: { model_name: id } };
+  }
+  if (/^gemini-/.test(id)) {
+    return { provider: "GOOGLE_GEMINI", provider_config: { model_name: id } };
+  }
+  if (/^grok-/.test(id)) {
+    return { provider: "XAI", provider_config: { model_name: id } };
+  }
+  throw new Error(`cannot infer a provider for model "${id}" \u2014 use a claude-*/gpt-*/gemini-*/grok-* cloud id ` + `or an org/name HuggingFace ONNX model path`);
+}
+async function ensureEmbeddingDimensions(config) {
+  if (config.provider !== "HF_TRANSFORMERS_ONNX")
+    return config;
+  if (config.provider_config.native_dimensions !== undefined)
+    return config;
+  const path = sanitizeHubRepoId(config.provider_config.model_path);
+  const url = `https://huggingface.co/${path}/resolve/main/config.json`;
+  const res = await fetch(url, { headers: hfAuthHeaders() });
+  if (!res.ok) {
+    throw new Error(`could not read ${path} config.json (${res.status}) to determine dimensions`);
+  }
+  const modelInfo = await res.json();
+  const dimensions = modelInfo.hidden_size ?? modelInfo.d_model;
+  if (!dimensions) {
+    throw new Error(`no hidden_size/d_model in ${path} config.json \u2014 cannot infer dimensions`);
+  }
+  return {
+    ...config,
+    provider_config: { ...config.provider_config, native_dimensions: dimensions }
+  };
+}
+async function ensureModelDownloaded(config) {
+  if (config.provider !== "LOCAL_LLAMACPP")
+    return;
+  if (config.provider_config.model_url === undefined)
+    return;
+  await downloadModel({ model: config });
+}
+function splitDtype(id) {
+  const colon = id.lastIndexOf(":");
+  if (colon <= id.indexOf("/"))
+    return [id, undefined];
+  return [id.slice(0, colon), id.slice(colon + 1)];
+}
+function parseModelList(models) {
+  return [
+    ...new Set(models.split(",").map((m) => m.trim()).filter((m) => m.length > 0))
+  ];
+}
+
+// src/evals/classify.ts
+import { Workflow } from "@workglow/task-graph";
+
+// src/evals/prompt.ts
+function fenceText(text) {
+  let fence = '"""';
+  while (text.includes(fence))
+    fence += '"';
+  return `${fence}
+${text}
+${fence}`;
+}
+
+// src/evals/classify.ts
+function expectedLabel(row, labelColumn, labelNames, candidateLabels = []) {
+  const raw = row[labelColumn];
+  if (typeof raw === "number" && Number.isInteger(raw)) {
+    const names = labelNames[labelColumn] ?? candidateLabels;
+    if (raw >= 0 && raw < names.length)
+      return names[raw];
+  }
+  return String(raw);
+}
+function resolveCandidateLabels(options, context) {
+  if (options.labels && options.labels.length > 0)
+    return options.labels;
+  const names = context.labelNames[options.labelColumn];
+  if (names && names.length > 0)
+    return names;
+  throw new Error(`no candidate labels: dataset column "${options.labelColumn}" has no ClassLabel names \u2014 ` + `pass --labels "a,b,c"`);
+}
+function buildClassifyPrompt(text, labels) {
+  return `Classify the following text into exactly one of these labels: ` + `${labels.join(", ")}.
+
+` + `Text:
+${fenceText(text)}
+
+` + `Respond with a JSON object of the form {"label": "<one of the labels>"}.`;
+}
+function makeClassifyExecutor(model, options, context) {
+  for (const column of [options.textColumn, options.labelColumn]) {
+    if (context.columns.length > 0 && !context.columns.includes(column)) {
+      throw new Error(`dataset has no column "${column}" (columns: ${context.columns.join(", ")}) \u2014 ` + `set --text-column/--label-column`);
+    }
+  }
+  const labels = resolveCandidateLabels(options, context);
+  const outputSchema = {
+    type: "object",
+    properties: { label: { type: "string", enum: [...labels] } },
+    required: ["label"],
+    additionalProperties: false
+  };
+  return async (row) => {
+    const text = String(row[options.textColumn] ?? "");
+    const workflow = new Workflow;
+    workflow.structuredGeneration({
+      model,
+      prompt: buildClassifyPrompt(text, labels),
+      outputSchema,
+      temperature: 0,
+      maxTokens: 256
+    });
+    const result = await workflow.run();
+    return {
+      expected: expectedLabel(row, options.labelColumn, context.labelNames, labels),
+      predicted: String(result.object?.label ?? "")
+    };
+  };
+}
+
+// src/evals/extract.ts
+import { Workflow as Workflow2 } from "@workglow/task-graph";
+var DEFAULT_INSTRUCTION = "Extract every entity mentioned in the text.";
+var UNSAFE_FIELDS = new Set(["__proto__", "constructor", "prototype"]);
+function parseExpectedRows(value, column) {
+  const parsed = typeof value === "string" ? JSON.parse(value) : value;
+  if (!Array.isArray(parsed) || parsed.some((row) => row === null || typeof row !== "object" || Array.isArray(row))) {
+    throw new Error(`column "${column}" must hold an array of objects (or its JSON string)`);
+  }
+  return parsed;
+}
+function resolveExtractionFields(expectedRows, keyField, fields) {
+  if (UNSAFE_FIELDS.has(keyField)) {
+    throw new Error(`--key-field "${keyField}" is not a usable field name`);
+  }
+  if (fields && fields.length > 0) {
+    const safe = fields.filter((field) => !UNSAFE_FIELDS.has(field));
+    return safe.includes(keyField) ? [...safe] : [keyField, ...safe];
+  }
+  const seen = new Set([keyField]);
+  for (const row of expectedRows) {
+    for (const key of Object.keys(row))
+      if (!UNSAFE_FIELDS.has(key))
+        seen.add(key);
+  }
+  return [...seen];
+}
+function buildExtractPrompt(text, instruction, keyField, fields) {
+  return `${instruction}
+
+` + `For each one, report the fields: ${fields.join(", ")}. ` + `"${keyField}" identifies the entity; use null when the text does not state a field.
+
+` + `Text:
+${fenceText(text)}
+
+` + `Respond with a JSON object of the form {"items": [{"${keyField}": "...", ...}, ...]}. ` + `Report each entity once; use only information from the text.`;
+}
+function buildItemsSchema(fields, keyField) {
+  const itemProperties = {};
+  for (const field of fields) {
+    itemProperties[field] = field === keyField ? { type: "string" } : { type: ["string", "null"] };
+  }
+  return {
+    type: "object",
+    properties: {
+      items: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: itemProperties,
+          required: [keyField],
+          additionalProperties: false
+        }
+      }
+    },
+    required: ["items"],
+    additionalProperties: false
+  };
+}
+function makeExtractExecutor(model, options, context) {
+  for (const column of [options.textColumn, options.expectedColumn]) {
+    if (context.columns.length > 0 && !context.columns.includes(column)) {
+      throw new Error(`dataset has no column "${column}" (columns: ${context.columns.join(", ")}) \u2014 ` + `set --text-column/--expected-column`);
+    }
+  }
+  const instruction = options.instruction ?? DEFAULT_INSTRUCTION;
+  const explicitFields = options.fields && options.fields.length > 0 ? resolveExtractionFields([], options.keyField, options.fields) : undefined;
+  const explicitSchema = explicitFields ? buildItemsSchema(explicitFields, options.keyField) : undefined;
+  return async (row) => {
+    const text = String(row[options.textColumn] ?? "");
+    const expectedRows = parseExpectedRows(row[options.expectedColumn], options.expectedColumn);
+    const fields = explicitFields ?? resolveExtractionFields(expectedRows, options.keyField, undefined);
+    const outputSchema = explicitSchema ?? buildItemsSchema(fields, options.keyField);
+    const workflow = new Workflow2;
+    workflow.structuredGeneration({
+      model,
+      prompt: buildExtractPrompt(text, instruction, options.keyField, fields),
+      outputSchema,
+      temperature: 0,
+      maxTokens: 2048
+    });
+    const result = await workflow.run();
+    const items = Array.isArray(result.object?.items) ? result.object.items : [];
+    return {
+      expected: JSON.stringify(expectedRows),
+      predicted: JSON.stringify(items)
+    };
+  };
+}
+
+// src/evals/similarity.ts
+import { Workflow as Workflow3 } from "@workglow/task-graph";
+import { cosineSimilarity } from "@workglow/util/schema";
+function makeSimilarityExecutor(model, options, context) {
+  for (const column of [options.textColumn, options.pairColumn, options.scoreColumn]) {
+    if (context.columns.length > 0 && !context.columns.includes(column)) {
+      throw new Error(`dataset has no column "${column}" (columns: ${context.columns.join(", ")}) \u2014 ` + `set --text-column/--pair-column/--score-column`);
+    }
+  }
+  return async (row) => {
+    const a = String(row[options.textColumn] ?? "");
+    const b = String(row[options.pairColumn] ?? "");
+    const workflow = new Workflow3;
+    workflow.textEmbedding({ model, text: [a, b] });
+    const result = await workflow.run();
+    const vectors = Array.isArray(result.vector) ? result.vector : [result.vector];
+    if (vectors.length < 2) {
+      throw new Error(`expected 2 embeddings, got ${vectors.length}`);
+    }
+    const gold = Number(row[options.scoreColumn]);
+    if (!Number.isFinite(gold)) {
+      throw new Error(`row has no numeric "${options.scoreColumn}" value (got ${JSON.stringify(row[options.scoreColumn])})`);
+    }
+    return {
+      expectedValue: gold,
+      predictedValue: cosineSimilarity(vectors[0], vectors[1])
+    };
+  };
+}
+
+// src/evals/runner.ts
+var EXECUTORS = {
+  classify: makeClassifyExecutor,
+  similarity: makeSimilarityExecutor,
+  extract: makeExtractExecutor
+};
+function makeExecutor(kind, model, columns, context) {
+  return EXECUTORS[kind](model, columns, context);
+}
+async function runSweep(stores, rows, options) {
+  const run = await stores.runs.put({
+    kind: options.kind,
+    dataset: options.dataset,
+    split: options.split,
+    models: JSON.stringify(options.models),
+    options: JSON.stringify(options.columns),
+    created_at: new Date().toISOString()
+  });
+  const parsedRows = rows.map((record) => {
+    try {
+      return { record, row: JSON.parse(record.data), parseError: undefined };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return { record, row: undefined, parseError: `invalid stored row: ${message}` };
+    }
+  });
+  const total = rows.length * options.models.length;
+  let done = 0;
+  for (const modelId of options.models) {
+    let executor;
+    let setupError;
+    try {
+      let config = resolveModelConfig(modelId, options.kind);
+      if (options.kind === "similarity")
+        config = await ensureEmbeddingDimensions(config);
+      await ensureModelDownloaded(config);
+      executor = makeExecutor(options.kind, config, options.columns, options.context);
+    } catch (err) {
+      setupError = err instanceof Error ? err.message : String(err);
+    }
+    for (const { record, row, parseError } of parsedRows) {
+      const t0 = performance.now();
+      let outcome;
+      if (!executor) {
+        outcome = failedOutcome(setupError ?? "executor unavailable");
+      } else if (!row) {
+        outcome = failedOutcome(parseError ?? "invalid stored row");
+      } else {
+        try {
+          const prediction = await executor(row);
+          outcome = {
+            ok: 1,
+            error: null,
+            expected: prediction.expected ?? null,
+            predicted: prediction.predicted ?? null,
+            expected_value: prediction.expectedValue ?? null,
+            predicted_value: prediction.predictedValue ?? null
+          };
+        } catch (err) {
+          outcome = failedOutcome(err instanceof Error ? err.message : String(err));
+        }
+      }
+      const latency = performance.now() - t0;
+      await stores.results.put({
+        run_id: run.run_id,
+        model: modelId,
+        row_index: record.row_index,
+        latency_ms: Math.round(latency * 100) / 100,
+        ...outcome
+      });
+      done++;
+      options.onProgress?.(done, total, modelId, outcome.ok === 1);
+    }
+  }
+  return run.run_id;
+}
+function failedOutcome(error) {
+  return {
+    ok: 0,
+    error,
+    expected: null,
+    predicted: null,
+    expected_value: null,
+    predicted_value: null
+  };
+}
+
+// src/commands/run.ts
+var DEFAULT_MODELS = {
+  classify: "onnx-community/LFM2.5-350M-ONNX",
+  similarity: "Xenova/all-MiniLM-L6-v2",
+  extract: "onnx-community/LFM2.5-350M-ONNX"
+};
+var DEFAULT_TEXT_COLUMN = {
+  classify: "text",
+  similarity: "sentence1",
+  extract: "text"
+};
+var DESCRIPTIONS = {
+  classify: "Classify each stored row with each model (StructuredGenerationTask) and score accuracy",
+  similarity: "Embed stored sentence pairs with each model (TextEmbeddingTask) and correlate cosine similarity with the gold score",
+  extract: "Extract structured records from each stored row (StructuredGenerationTask) and score field agreement, entity recall, and precision against the gold rows"
+};
+function registerRunCommand(program, openStores, ensureProviders) {
+  for (const kind of ["classify", "similarity", "extract"]) {
+    const command = program.command(`run-${kind}`).description(DESCRIPTIONS[kind]).requiredOption("--dataset <id>", "a dataset previously stored via `dataset pull`").option("--split <split>", "dataset split", "test").option("--models <list>", `comma-separated model ids (default: ${DEFAULT_MODELS[kind]})`).option("--limit <n>", "cap the number of stored rows to run").option("--text-column <name>", `input text column (default: ${DEFAULT_TEXT_COLUMN[kind]})`).option("--format <fmt>", "report output: table or json", "table");
+    if (kind === "classify") {
+      command.option("--label-column <name>", "gold label column (default: label)").option("--labels <list>", "comma-separated candidate labels override");
+    } else if (kind === "similarity") {
+      command.option("--pair-column <name>", "second sentence column (default: sentence2)").option("--score-column <name>", "gold score column (default: score)");
+    } else {
+      command.option("--expected-column <name>", "gold column: array of objects or JSON (default: expected)").option("--key-field <name>", "field that identifies an entity when aligning (default: name)").option("--fields <list>", "comma-separated fields to extract and score").option("--instruction <text>", "task sentence at the top of the prompt");
+    }
+    command.action(async (flags) => {
+      try {
+        await ensureProviders();
+        await runEval(kind, flags, await openStores());
+      } catch (err) {
+        console.error(`Error: ${formatError(err)}`);
+        process.exitCode = 1;
+      }
+    });
+  }
+}
+function parseNameList(value, flag) {
+  if (value === undefined)
+    return;
+  const names = [
+    ...new Set(value.split(",").map((name) => name.trim()).filter((name) => name.length > 0))
+  ];
+  if (names.length === 0)
+    throw new Error(`${flag} was given but contains no names`);
+  return names;
+}
+async function runEval(kind, flags, stores) {
+  const meta = await stores.datasets.get({ dataset: flags.dataset, split: flags.split });
+  if (!meta) {
+    throw new Error(`dataset ${flags.dataset} [${flags.split}] is not in storage \u2014 ` + `run \`workglow-eval dataset pull ${flags.dataset} --split ${flags.split}\` first`);
+  }
+  const rows = await stores.rows.query({ dataset: flags.dataset, split: flags.split }) ?? [];
+  rows.sort((a, b) => a.row_index - b.row_index);
+  const limit = flags.limit === undefined ? undefined : parseIntFlag(flags.limit, "--limit", 1);
+  const limited = limit === undefined ? rows : rows.slice(0, limit);
+  if (limited.length === 0)
+    throw new Error("no stored rows to evaluate");
+  const models = parseModelList(flags.models ?? DEFAULT_MODELS[kind]);
+  const keyField = (flags.keyField ?? "name").trim();
+  if (keyField.length === 0)
+    throw new Error("--key-field must not be empty");
+  const columns = {
+    textColumn: flags.textColumn ?? DEFAULT_TEXT_COLUMN[kind],
+    labelColumn: flags.labelColumn ?? "label",
+    labels: parseNameList(flags.labels, "--labels"),
+    pairColumn: flags.pairColumn ?? "sentence2",
+    scoreColumn: flags.scoreColumn ?? "score",
+    expectedColumn: flags.expectedColumn ?? "expected",
+    keyField,
+    fields: parseNameList(flags.fields, "--fields"),
+    instruction: flags.instruction
+  };
+  const context = {
+    columns: JSON.parse(meta.columns),
+    labelNames: meta.label_names ? JSON.parse(meta.label_names) : {}
+  };
+  console.error(`running ${kind} over ${limited.length} rows of ${flags.dataset} [${flags.split}] ` + `with ${models.length} model(s)`);
+  const runId = await runSweep(stores, limited, {
+    kind,
+    dataset: flags.dataset,
+    split: flags.split,
+    models,
+    columns,
+    context,
+    onProgress: (done, total, model, ok) => {
+      console.error(`[${done}/${total}] ${model} ${ok ? "ok" : "FAIL"}`);
+    }
+  });
+  console.error(`run ${runId} complete
+`);
+  await printReport(stores, runId, flags.format);
+}
+
+// src/providers.ts
+import { registerAiTasks } from "@workglow/ai";
+import { registerAnthropicInline } from "@workglow/anthropic/ai-runtime";
+import { registerGeminiInline } from "@workglow/google-gemini/ai-runtime";
+import { registerHuggingFaceTransformers } from "@workglow/huggingface-transformers/ai";
+import { registerLlamaCpp } from "@workglow/node-llama-cpp/ai";
+import { registerOpenAiInline } from "@workglow/openai/ai-runtime";
+import { registerBaseTasks, registerBuiltInTransforms } from "@workglow/task-graph";
+import { EnvCredentialStore, setGlobalCredentialStore } from "@workglow/util";
+import { registerXaiInline } from "@workglow/xai/ai-runtime";
+async function registerEvalProviders(config) {
+  registerBaseTasks();
+  registerAiTasks();
+  registerBuiltInTransforms();
+  setGlobalCredentialStore(new EnvCredentialStore);
+  process.env.WORKGLOW_MODEL_CACHE = config.modelCache;
+  const registrations = [
+    [
+      "HF_TRANSFORMERS_ONNX",
+      () => registerHuggingFaceTransformers({
+        worker: () => new Worker(new URL("./worker_hft.js", import.meta.url), { type: "module" })
+      })
+    ],
+    [
+      "LOCAL_LLAMACPP",
+      () => registerLlamaCpp({
+        worker: () => new Worker(new URL("./worker_llamacpp.js", import.meta.url), { type: "module" })
+      })
+    ],
+    ["ANTHROPIC", () => registerAnthropicInline()],
+    ["OPENAI", () => registerOpenAiInline()],
+    ["GOOGLE_GEMINI", () => registerGeminiInline()],
+    ["XAI", () => registerXaiInline()]
+  ];
+  for (const [name, register] of registrations) {
+    try {
+      await register();
+    } catch (err) {
+      console.error(`warning: provider ${name} failed to register and its models will be unavailable: ` + `${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+}
+
+// src/storage.ts
+import { Sqlite, SqliteTabularStorage } from "@workglow/sqlite/storage";
+import { InMemoryTabularStorage } from "@workglow/storage";
+var DatasetMetaSchema = {
+  type: "object",
+  properties: {
+    dataset: { type: "string" },
+    split: { type: "string" },
+    config: { type: "string" },
+    num_rows: { type: "number" },
+    columns: { type: "string" },
+    label_names: { type: ["string", "null"] },
+    source: { type: "string" },
+    fetched_at: { type: "string" }
+  },
+  required: ["dataset", "split", "config", "num_rows", "columns", "source", "fetched_at"],
+  additionalProperties: false
+};
+var DatasetMetaPrimaryKeyNames = ["dataset", "split"];
+var DatasetRowSchema = {
+  type: "object",
+  properties: {
+    dataset: { type: "string" },
+    split: { type: "string" },
+    row_index: { type: "number" },
+    data: { type: "string" }
+  },
+  required: ["dataset", "split", "row_index", "data"],
+  additionalProperties: false
+};
+var DatasetRowPrimaryKeyNames = ["dataset", "split", "row_index"];
+var EvalRunSchema = {
+  type: "object",
+  properties: {
+    run_id: { type: "string", "x-auto-generated": true },
+    kind: { type: "string" },
+    dataset: { type: "string" },
+    split: { type: "string" },
+    models: { type: "string" },
+    options: { type: "string" },
+    created_at: { type: "string" }
+  },
+  required: ["run_id", "kind", "dataset", "split", "models", "options", "created_at"],
+  additionalProperties: false
+};
+var EvalRunPrimaryKeyNames = ["run_id"];
+var EvalResultSchema = {
+  type: "object",
+  properties: {
+    run_id: { type: "string" },
+    model: { type: "string" },
+    row_index: { type: "number" },
+    ok: { type: "number" },
+    error: { type: ["string", "null"] },
+    expected: { type: ["string", "null"] },
+    predicted: { type: ["string", "null"] },
+    expected_value: { type: ["number", "null"] },
+    predicted_value: { type: ["number", "null"] },
+    latency_ms: { type: "number" }
+  },
+  required: ["run_id", "model", "row_index", "ok", "latency_ms"],
+  additionalProperties: false
+};
+var EvalResultPrimaryKeyNames = ["run_id", "model", "row_index"];
+async function createSqliteStores(config) {
+  await Sqlite.init();
+  const db = new Sqlite.Database(config.dbPath);
+  const stores = {
+    datasets: new SqliteTabularStorage(db, "eval_dataset", DatasetMetaSchema, DatasetMetaPrimaryKeyNames),
+    rows: new SqliteTabularStorage(db, "eval_dataset_row", DatasetRowSchema, DatasetRowPrimaryKeyNames),
+    runs: new SqliteTabularStorage(db, "eval_run", EvalRunSchema, EvalRunPrimaryKeyNames),
+    results: new SqliteTabularStorage(db, "eval_result", EvalResultSchema, EvalResultPrimaryKeyNames)
+  };
+  await setupStores(stores);
+  return stores;
+}
+async function setupStores(stores) {
+  await stores.datasets.setupDatabase();
+  await stores.rows.setupDatabase();
+  await stores.runs.setupDatabase();
+  await stores.results.setupDatabase();
+}
+
+// src/workglow-eval.ts
+var config = loadConfig();
+var stores;
+var openStores = () => stores ??= createSqliteStores(config);
+var providers;
+var ensureProviders = () => providers ??= registerEvalProviders(config);
+program.version("2.0.0").description("Workglow eval example \u2014 pull HuggingFace datasets into storage, run task workflows " + "across models, and score the stored results");
+registerDatasetCommand(program, openStores);
+registerRunCommand(program, openStores, ensureProviders);
+registerReportCommand(program, openStores);
+await program.parseAsync(process.argv);
+process.exit(process.exitCode ?? 0);