@deepagents/evals 0.19.0

package/dist/index.js

@@ -0,0 +1,1763 @@
+// packages/evals/src/dataset/index.ts
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { extname } from "node:path";
+import { createInterface } from "node:readline";
+
+// packages/evals/src/dataset/hf.ts
+var HF_BASE_URL = "https://datasets-server.huggingface.co/rows";
+var PAGE_SIZE = 100;
+function hf(options) {
+  return {
+    [Symbol.asyncIterator]() {
+      return paginate(options);
+    }
+  };
+}
+async function* paginate(options) {
+  const { dataset: dataset2, config, split, rows } = options;
+  const limit = rows ?? Infinity;
+  let offset = 0;
+  let yielded = 0;
+  while (yielded < limit) {
+    const pageSize = limit === Infinity ? PAGE_SIZE : Math.min(PAGE_SIZE, limit - yielded);
+    const url = buildUrl(dataset2, config, split, offset, pageSize);
+    const page = await fetchPage(url);
+    if (page.rows.length === 0) return;
+    for (const entry of page.rows) {
+      yield entry.row;
+      yielded++;
+      if (yielded >= limit) return;
+    }
+    offset += page.rows.length;
+    if (page.rows.length < pageSize || offset >= page.num_rows_total) return;
+  }
+}
+function buildUrl(dataset2, config, split, offset, length) {
+  const url = new URL(HF_BASE_URL);
+  url.searchParams.set("dataset", dataset2);
+  url.searchParams.set("config", config);
+  url.searchParams.set("split", split);
+  url.searchParams.set("offset", String(offset));
+  url.searchParams.set("length", String(length));
+  return url.toString();
+}
+async function fetchPage(url) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw new Error(
+      `HuggingFace API error ${response.status}: ${body || response.statusText}`
+    );
+  }
+  const text = await response.text();
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(
+      `HuggingFace API returned non-JSON response from ${url}: ${text.slice(0, 200)}`
+    );
+  }
+}
+
+// packages/evals/src/dataset/index.ts
+var Dataset = class _Dataset {
+  #source;
+  constructor(source) {
+    this.#source = source;
+  }
+  map(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        yield fn(item);
+      }
+    });
+  }
+  filter(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        if (fn(item)) yield item;
+      }
+    });
+  }
+  limit(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      let count = 0;
+      for await (const item of source()) {
+        if (count >= n) return;
+        yield item;
+        count++;
+      }
+    });
+  }
+  shuffle() {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      for (let i = items.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      yield* items;
+    });
+  }
+  sample(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      const count = Math.min(Math.max(0, n), items.length);
+      for (let i = items.length - 1; i > items.length - count - 1; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      for (let i = items.length - count; i < items.length; i++) {
+        yield items[i];
+      }
+    });
+  }
+  async toArray() {
+    const result = [];
+    for await (const item of this.#source()) {
+      result.push(item);
+    }
+    return result;
+  }
+  [Symbol.asyncIterator]() {
+    return this.#source()[Symbol.asyncIterator]();
+  }
+};
+function parseCSVLine(line) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += char;
+      }
+    } else {
+      if (char === '"' && current === "") {
+        inQuotes = true;
+      } else if (char === ",") {
+        fields.push(current);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+  }
+  fields.push(current);
+  return fields;
+}
+function loadJSON(filePath) {
+  return async function* () {
+    const content = await readFile(filePath, "utf-8");
+    const data = JSON.parse(content);
+    if (!Array.isArray(data)) {
+      throw new Error(`JSON file "${filePath}" does not contain an array`);
+    }
+    yield* data;
+  };
+}
+function loadJSONL(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (trimmed) {
+          yield JSON.parse(trimmed);
+        }
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function loadCSV(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      let headers;
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        const fields = parseCSVLine(trimmed);
+        if (!headers) {
+          headers = fields;
+          continue;
+        }
+        const row = {};
+        for (let i = 0; i < headers.length; i++) {
+          row[headers[i]] = fields[i] ?? "";
+        }
+        yield row;
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function dataset(source) {
+  if (Array.isArray(source)) {
+    return new Dataset(async function* () {
+      yield* source;
+    });
+  }
+  if (typeof source === "object" && Symbol.asyncIterator in source) {
+    return new Dataset(() => source);
+  }
+  const ext = extname(source).toLowerCase();
+  switch (ext) {
+    case ".json":
+      return new Dataset(loadJSON(source));
+    case ".jsonl":
+      return new Dataset(loadJSONL(source));
+    case ".csv":
+      return new Dataset(loadCSV(source));
+    default:
+      throw new Error(
+        `Unsupported file extension "${ext}" for dataset file "${source}". Supported: .json, .jsonl, .csv`
+      );
+  }
+}
+
+// packages/evals/src/scorers/index.ts
+import { generateObject } from "ai";
+import { z } from "zod";
+var exactMatch = async ({ output, expected }) => {
+  const exp = expected == null ? "" : String(expected);
+  if (output === exp) return { score: 1 };
+  return {
+    score: 0,
+    reason: `Output does not exactly match expected. Expected "${exp}" but got "${output}".`
+  };
+};
+var includes = async ({ output, expected }) => {
+  const exp = expected == null ? "" : String(expected);
+  if (output.includes(exp)) return { score: 1 };
+  return {
+    score: 0,
+    reason: `Output does not include expected substring "${exp}".`
+  };
+};
+function regex(pattern) {
+  return async ({ output }) => {
+    return { score: pattern.test(output) ? 1 : 0 };
+  };
+}
+function levenshteinDistance(a, b) {
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+  if (a.length > b.length) [a, b] = [b, a];
+  let prev = Array.from({ length: a.length + 1 }, (_, i) => i);
+  let curr = new Array(a.length + 1);
+  for (let j = 1; j <= b.length; j++) {
+    curr[0] = j;
+    for (let i = 1; i <= a.length; i++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr[i] = Math.min(prev[i] + 1, curr[i - 1] + 1, prev[i - 1] + cost);
+    }
+    [prev, curr] = [curr, prev];
+  }
+  return prev[a.length];
+}
+var levenshtein = async ({ output, expected }) => {
+  const exp = expected == null ? "" : String(expected);
+  if (output.length === 0 && exp.length === 0) return { score: 1 };
+  const maxLen = Math.max(output.length, exp.length);
+  const distance = levenshteinDistance(output, exp);
+  const score = Math.max(0, 1 - distance / maxLen);
+  if (score === 1) return { score };
+  return {
+    score,
+    reason: `Levenshtein distance is ${distance} across max length ${maxLen}.`
+  };
+};
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a)) {
+    if (!Array.isArray(b) || a.length !== b.length) return false;
+    return a.every((val, i) => deepEqual(val, b[i]));
+  }
+  if (typeof a === "object") {
+    const keysA = Object.keys(a).sort();
+    const keysB = Object.keys(b).sort();
+    if (keysA.length !== keysB.length) return false;
+    return keysA.every(
+      (key, i) => keysB[i] === key && deepEqual(
+        a[key],
+        b[key]
+      )
+    );
+  }
+  return false;
+}
+var jsonMatch = async ({ output, expected }) => {
+  try {
+    const parsedOutput = JSON.parse(output);
+    const parsedExpected = typeof expected === "string" ? JSON.parse(expected) : expected;
+    if (deepEqual(parsedOutput, parsedExpected)) return { score: 1 };
+    return { score: 0, reason: "JSON payload differs from expected JSON." };
+  } catch {
+    return { score: 0, reason: "Failed to parse JSON" };
+  }
+};
+var llmScorerSchema = z.object({
+  score: z.number().min(0).max(1),
+  reason: z.string()
+});
+function llmJudge(config) {
+  return async ({ input, output, expected }) => {
+    const { object } = await generateObject({
+      model: config.model,
+      schema: llmScorerSchema,
+      prompt: `You are an expert evaluator. Grade the output based on the following criteria:
+${config.criteria}
+
+Input: ${JSON.stringify(input)}
+Output: ${output}
+${expected != null ? `Expected: ${JSON.stringify(expected)}` : ""}
+
+Return a score from 0.0 to 1.0 and a brief reason.`
+    });
+    return { score: object.score, reason: object.reason };
+  };
+}
+function factuality(config) {
+  return async ({ input, output, expected }) => {
+    const { object } = await generateObject({
+      model: config.model,
+      schema: llmScorerSchema,
+      prompt: `You are a factuality evaluator. Determine whether the output is factually consistent with the expected reference.
+
+Input: ${JSON.stringify(input)}
+Output: ${output}
+Expected reference: ${JSON.stringify(expected)}
+
+Score 1.0 if the output is factually consistent with the reference, 0.0 if it contradicts it. Use intermediate scores for partial consistency.
+Return a score from 0.0 to 1.0 and a brief reason.`
+    });
+    return { score: object.score, reason: object.reason };
+  };
+}
+function all(...scorers) {
+  return async (args) => {
+    if (scorers.length === 0) return { score: 1 };
+    const results = await Promise.all(scorers.map((s) => s(args)));
+    const minResult = results.reduce(
+      (min, r) => r.score < min.score ? r : min
+    );
+    const reasons = results.filter((r) => r.reason).map((r) => r.reason).join("; ");
+    return { score: minResult.score, reason: reasons || void 0 };
+  };
+}
+function any(...scorers) {
+  return async (args) => {
+    if (scorers.length === 0) return { score: 0 };
+    const results = await Promise.all(scorers.map((s) => s(args)));
+    const maxResult = results.reduce(
+      (max, r) => r.score > max.score ? r : max
+    );
+    return { score: maxResult.score, reason: maxResult.reason };
+  };
+}
+function weighted(config) {
+  return async (args) => {
+    const entries = Object.entries(config);
+    const results = await Promise.all(
+      entries.map(async ([name, { scorer, weight }]) => ({
+        name,
+        result: await scorer(args),
+        weight
+      }))
+    );
+    const totalWeight = results.reduce((sum, r) => sum + r.weight, 0);
+    const weightedScore = results.reduce(
+      (sum, r) => sum + r.result.score * r.weight,
+      0
+    );
+    const score = totalWeight > 0 ? weightedScore / totalWeight : 0;
+    const reasons = results.map((r) => `${r.name}: ${r.result.score.toFixed(2)} (w=${r.weight})`).join(", ");
+    return { score, reason: reasons || void 0 };
+  };
+}
+
+// packages/evals/src/store/index.ts
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+// packages/evals/src/store/ddl.sqlite.sql
+var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
+
+// packages/evals/src/store/index.ts
+var RunStore = class {
+  #db;
+  #statements = /* @__PURE__ */ new Map();
+  #stmt(sql) {
+    let stmt = this.#statements.get(sql);
+    if (!stmt) {
+      stmt = this.#db.prepare(sql);
+      this.#statements.set(sql, stmt);
+    }
+    return stmt;
+  }
+  #transaction(fn) {
+    this.#db.exec("BEGIN TRANSACTION");
+    try {
+      const result = fn();
+      this.#db.exec("COMMIT");
+      return result;
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+  constructor(pathOrDb) {
+    if (pathOrDb instanceof DatabaseSync) {
+      this.#db = pathOrDb;
+    } else {
+      const dbPath = pathOrDb ?? ".evals/store.db";
+      mkdirSync(dirname(dbPath), { recursive: true });
+      this.#db = new DatabaseSync(dbPath);
+    }
+    this.#db.exec(ddl_sqlite_default);
+    this.#migrateRunsTableToSuiteRequired();
+    this.#migratePromptsTableIfNeeded();
+    this.#db.exec(
+      "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+    );
+  }
+  #migratePromptsTableIfNeeded() {
+    const columns = this.#stmt("PRAGMA table_info(prompts)").all();
+    if (columns.length === 0) return;
+    if (columns.some((column) => column.name === "version")) return;
+    this.#transaction(() => {
+      this.#db.exec("ALTER TABLE prompts RENAME TO prompts_legacy");
+      this.#db.exec(`
+        CREATE TABLE prompts (
+          id TEXT PRIMARY KEY,
+          name TEXT NOT NULL,
+          version INTEGER NOT NULL,
+          content TEXT NOT NULL,
+          created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+          UNIQUE(name, version)
+        )
+      `);
+      this.#db.exec(`
+        INSERT INTO prompts (id, name, version, content, created_at)
+        SELECT id, name, 1, content, created_at
+        FROM prompts_legacy
+      `);
+      this.#db.exec("DROP TABLE prompts_legacy");
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at)"
+      );
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+      );
+    });
+  }
+  #migrateRunsTableToSuiteRequired() {
+    const runColumns = this.#stmt("PRAGMA table_info(runs)").all();
+    if (runColumns.length === 0) return;
+    const suiteColumn = runColumns.find((column) => column.name === "suite_id");
+    const hasNonNullSuite = suiteColumn?.notnull === 1;
+    const runForeignKeys = this.#stmt(
+      "PRAGMA foreign_key_list(runs)"
+    ).all();
+    const suiteForeignKey = runForeignKeys.find(
+      (fk) => fk.from === "suite_id" && fk.table === "suites"
+    );
+    const hasCascadeDelete = suiteForeignKey?.on_delete === "CASCADE";
+    if (hasNonNullSuite && hasCascadeDelete) return;
+    this.#statements.clear();
+    this.#transaction(() => {
+      this.#db.exec(`
+        CREATE TABLE runs_next (
+          id TEXT PRIMARY KEY,
+          suite_id TEXT NOT NULL,
+          name TEXT NOT NULL,
+          model TEXT NOT NULL,
+          config TEXT,
+          started_at INTEGER NOT NULL,
+          finished_at INTEGER,
+          status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),
+          summary TEXT,
+          FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE
+        )
+      `);
+      this.#db.exec("DELETE FROM runs WHERE suite_id IS NULL");
+      this.#db.exec(`
+        INSERT INTO runs_next (id, suite_id, name, model, config, started_at, finished_at, status, summary)
+        SELECT r.id, r.suite_id, r.name, r.model, r.config, r.started_at, r.finished_at, r.status, r.summary
+        FROM runs r
+        JOIN suites s ON s.id = r.suite_id
+      `);
+      this.#db.exec("DROP TABLE runs");
+      this.#db.exec("ALTER TABLE runs_next RENAME TO runs");
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id)"
+      );
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at)"
+      );
+    });
+    this.#statements.clear();
+  }
+  createSuite(name) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    this.#stmt(
+      "INSERT INTO suites (id, name, created_at) VALUES (?, ?, ?)"
+    ).run(id, name, now);
+    return { id, name, created_at: now };
+  }
+  createRun(run) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    this.#stmt(
+      "INSERT INTO runs (id, suite_id, name, model, config, started_at) VALUES (?, ?, ?, ?, ?, ?)"
+    ).run(
+      id,
+      run.suite_id,
+      run.name,
+      run.model,
+      run.config ? JSON.stringify(run.config) : null,
+      now
+    );
+    return id;
+  }
+  finishRun(runId, status, summary) {
+    this.#stmt(
+      "UPDATE runs SET finished_at = ?, status = ?, summary = ? WHERE id = ?"
+    ).run(Date.now(), status, summary ? JSON.stringify(summary) : null, runId);
+  }
+  saveCases(cases) {
+    this.#transaction(() => {
+      const stmt = this.#stmt(
+        "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+      );
+      for (const c of cases) {
+        stmt.run(
+          c.id,
+          c.run_id,
+          c.idx,
+          JSON.stringify(c.input),
+          c.output,
+          c.expected != null ? JSON.stringify(c.expected) : null,
+          c.latency_ms,
+          c.tokens_in,
+          c.tokens_out,
+          c.error ?? null
+        );
+      }
+    });
+  }
+  saveScores(scores) {
+    this.#transaction(() => {
+      const stmt = this.#stmt(
+        "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
+      );
+      for (const s of scores) {
+        stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
+      }
+    });
+  }
+  getRun(runId) {
+    const row = this.#stmt("SELECT * FROM runs WHERE id = ?").get(runId);
+    if (!row) return void 0;
+    return {
+      id: row.id,
+      suite_id: row.suite_id,
+      name: row.name,
+      model: row.model,
+      config: row.config ? JSON.parse(row.config) : null,
+      started_at: row.started_at,
+      finished_at: row.finished_at,
+      status: row.status,
+      summary: row.summary ? JSON.parse(row.summary) : null
+    };
+  }
+  listRuns(suiteId) {
+    const sql = suiteId ? "SELECT * FROM runs WHERE suite_id = ? ORDER BY started_at" : "SELECT * FROM runs ORDER BY started_at";
+    const rows = suiteId ? this.#stmt(sql).all(suiteId) : this.#stmt(sql).all();
+    return rows.map((row) => ({
+      id: row.id,
+      suite_id: row.suite_id,
+      name: row.name,
+      model: row.model,
+      config: row.config ? JSON.parse(row.config) : null,
+      started_at: row.started_at,
+      finished_at: row.finished_at,
+      status: row.status,
+      summary: row.summary ? JSON.parse(row.summary) : null
+    }));
+  }
+  getCases(runId) {
+    const rows = this.#stmt(
+      "SELECT * FROM cases WHERE run_id = ? ORDER BY idx"
+    ).all(runId);
+    return rows.map((row) => ({
+      id: row.id,
+      run_id: row.run_id,
+      idx: row.idx,
+      input: JSON.parse(row.input),
+      output: row.output,
+      expected: row.expected ? JSON.parse(row.expected) : null,
+      latency_ms: row.latency_ms,
+      tokens_in: row.tokens_in,
+      tokens_out: row.tokens_out,
+      error: row.error
+    }));
+  }
+  getFailingCases(runId, threshold = 0.5) {
+    const rows = this.#stmt(
+      `SELECT c.*, s.scorer_name, s.score, s.reason as score_reason
+       FROM cases c
+       JOIN scores s ON s.case_id = c.id
+       WHERE c.run_id = ? AND s.score < ?
+       ORDER BY c.idx`
+    ).all(runId, threshold);
+    const caseMap = /* @__PURE__ */ new Map();
+    for (const row of rows) {
+      let c = caseMap.get(row.id);
+      if (!c) {
+        c = {
+          id: row.id,
+          run_id: row.run_id,
+          idx: row.idx,
+          input: JSON.parse(row.input),
+          output: row.output,
+          expected: row.expected ? JSON.parse(row.expected) : null,
+          latency_ms: row.latency_ms,
+          tokens_in: row.tokens_in,
+          tokens_out: row.tokens_out,
+          error: row.error,
+          scores: []
+        };
+        caseMap.set(row.id, c);
+      }
+      c.scores.push({
+        scorer_name: row.scorer_name,
+        score: row.score,
+        reason: row.score_reason
+      });
+    }
+    return Array.from(caseMap.values());
+  }
+  getRunSummary(runId, threshold = 0.5) {
+    const totals = this.#stmt(
+      `SELECT
+        COUNT(DISTINCT c.id) as totalCases,
+        COALESCE(SUM(c.latency_ms), 0) as totalLatencyMs,
+        COALESCE(SUM(c.tokens_in), 0) as totalTokensIn,
+        COALESCE(SUM(c.tokens_out), 0) as totalTokensOut
+       FROM cases c WHERE c.run_id = ?`
+    ).get(runId);
+    const scorerMeans = this.#stmt(
+      `SELECT s.scorer_name, AVG(s.score) as meanScore
+       FROM scores s
+       JOIN cases c ON c.id = s.case_id
+       WHERE c.run_id = ?
+       GROUP BY s.scorer_name`
+    ).all(runId);
+    const meanScores = {};
+    for (const row of scorerMeans) {
+      meanScores[row.scorer_name] = row.meanScore;
+    }
+    const passFail = this.#stmt(
+      `SELECT c.id,
+        MIN(s.score) as minScore
+       FROM cases c
+       JOIN scores s ON s.case_id = c.id
+       WHERE c.run_id = ?
+       GROUP BY c.id`
+    ).all(runId);
+    let passCount = 0;
+    let failCount = 0;
+    for (const row of passFail) {
+      if (row.minScore >= threshold) passCount++;
+      else failCount++;
+    }
+    return {
+      totalCases: totals.totalCases,
+      passCount,
+      failCount,
+      meanScores,
+      totalLatencyMs: totals.totalLatencyMs,
+      totalTokensIn: totals.totalTokensIn,
+      totalTokensOut: totals.totalTokensOut
+    };
+  }
+  listSuites() {
+    const rows = this.#stmt(
+      "SELECT * FROM suites ORDER BY created_at DESC"
+    ).all();
+    return rows.map((row) => ({
+      id: row.id,
+      name: row.name,
+      created_at: row.created_at
+    }));
+  }
+  createPrompt(name, content) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    const latest = this.#stmt(
+      "SELECT MAX(version) as latestVersion FROM prompts WHERE name = ?"
+    ).get(name);
+    const version = (latest?.latestVersion ?? 0) + 1;
+    this.#stmt(
+      "INSERT INTO prompts (id, name, version, content, created_at) VALUES (?, ?, ?, ?, ?)"
+    ).run(id, name, version, content, now);
+    return { id, name, version, content, created_at: now };
+  }
+  listPrompts() {
+    const rows = this.#stmt(
+      "SELECT * FROM prompts ORDER BY name COLLATE NOCASE ASC, version DESC"
+    ).all();
+    return rows.map((row) => ({
+      id: row.id,
+      name: row.name,
+      version: row.version,
+      content: row.content,
+      created_at: row.created_at
+    }));
+  }
+  getPrompt(id) {
+    const row = this.#stmt("SELECT * FROM prompts WHERE id = ?").get(id);
+    if (!row) return void 0;
+    return {
+      id: row.id,
+      name: row.name,
+      version: row.version,
+      content: row.content,
+      created_at: row.created_at
+    };
+  }
+  deletePrompt(id) {
+    this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
+  }
+};
+
+// packages/evals/src/engine/index.ts
+import { EventEmitter } from "node:events";
+var EvalEmitter = class extends EventEmitter {
+  on(event, listener) {
+    return super.on(event, listener);
+  }
+  emit(event, data) {
+    return super.emit(event, data);
+  }
+};
+function errorMessage(err) {
+  if (err instanceof Error) {
+    return `${err.name}: ${err.message}`;
+  }
+  if (typeof err === "string") return err;
+  if (err == null) return "Unknown error";
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return String(err);
+  }
+}
+function serializeError(err) {
+  if (err instanceof Error) {
+    return JSON.stringify({
+      name: err.name,
+      message: err.message,
+      stack: err.stack,
+      cause: err.cause instanceof Error ? {
+        name: err.cause.name,
+        message: err.cause.message
+      } : err.cause
+    });
+  }
+  if (typeof err === "string") return JSON.stringify({ message: err });
+  if (err == null) return JSON.stringify({ message: "Unknown error" });
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return JSON.stringify({ message: String(err) });
+  }
+}
+function failureScores(scorerNames, error) {
+  const reason = `Task failed: ${errorMessage(error)}`;
+  const scores = {};
+  for (const scorerName of scorerNames) {
+    scores[scorerName] = { score: 0, reason };
+  }
+  return scores;
+}
+function createSemaphore(maxConcurrency) {
+  let active = 0;
+  const queue = [];
+  return {
+    async acquire() {
+      if (active < maxConcurrency) {
+        active++;
+        return;
+      }
+      return new Promise((resolve) => queue.push(resolve));
+    },
+    release() {
+      active--;
+      const next = queue.shift();
+      if (next) {
+        active++;
+        next();
+      }
+    }
+  };
+}
+async function wrapTask(task, input, timeoutMs) {
+  const start = performance.now();
+  let timerId;
+  try {
+    const result = await Promise.race([
+      task(input),
+      new Promise((_, reject) => {
+        timerId = setTimeout(
+          () => reject(new Error("timeout exceeded")),
+          timeoutMs
+        );
+      })
+    ]);
+    clearTimeout(timerId);
+    const latencyMs = Math.round(performance.now() - start);
+    return {
+      output: result.output,
+      latencyMs,
+      tokensIn: result.usage?.inputTokens ?? 0,
+      tokensOut: result.usage?.outputTokens ?? 0
+    };
+  } catch (err) {
+    clearTimeout(timerId);
+    const latencyMs = Math.round(performance.now() - start);
+    return {
+      output: "",
+      latencyMs,
+      tokensIn: 0,
+      tokensOut: 0,
+      error: err
+    };
+  }
+}
+function clampScore(score, scorerName) {
+  if (score < 0 || score > 1) {
+    console.warn(
+      `Scorer "${scorerName}" returned out-of-range score ${score}, clamping to 0..1`
+    );
+    return Math.max(0, Math.min(1, score));
+  }
+  return score;
+}
+async function runEval(config) {
+  const {
+    name,
+    model,
+    dataset: ds,
+    task,
+    scorers,
+    store,
+    suiteId,
+    maxConcurrency = 10,
+    batchSize,
+    timeout = 3e4,
+    trials = 1,
+    threshold = 0.5
+  } = config;
+  const emitter = config.emitter ?? new EvalEmitter();
+  const resolvedSuiteId = suiteId ?? store.createSuite(name).id;
+  const runId = store.createRun({
+    suite_id: resolvedSuiteId,
+    name,
+    model,
+    config: config.config
+  });
+  const items = [];
+  let idx = 0;
+  for await (const item of ds) {
+    items.push({ index: idx++, input: item });
+  }
+  emitter.emit("run:start", { runId, totalCases: items.length, name, model });
+  const semaphore = createSemaphore(maxConcurrency);
+  const scorerNames = Object.keys(scorers);
+  const allCaseScores = [];
+  const processItem = async ({ index, input }) => {
+    await semaphore.acquire();
+    try {
+      emitter.emit("case:start", { runId, index, input });
+      let finalResult;
+      let finalScores;
+      if (trials > 1) {
+        const trialResults = [];
+        for (let t = 0; t < trials; t++) {
+          const result = await wrapTask(task, input, timeout);
+          if (result.error) {
+            trialResults.push({
+              result,
+              scores: failureScores(scorerNames, result.error)
+            });
+          } else {
+            const scores = {};
+            for (const [sName, scorer] of Object.entries(scorers)) {
+              const sr = await scorer({
+                input,
+                output: result.output,
+                expected: input.expected
+              });
+              scores[sName] = {
+                score: clampScore(sr.score, sName),
+                reason: sr.reason
+              };
+            }
+            trialResults.push({ result, scores });
+          }
+        }
+        const lastSuccessful = [...trialResults].reverse().find((t) => !t.result.error);
+        const baseResult = lastSuccessful?.result ?? trialResults[trialResults.length - 1].result;
+        finalResult = {
+          output: baseResult.output,
+          latencyMs: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.latencyMs, 0) / trials
+          ),
+          tokensIn: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.tokensIn, 0) / trials
+          ),
+          tokensOut: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.tokensOut, 0) / trials
+          ),
+          error: lastSuccessful ? void 0 : baseResult.error
+        };
+        finalScores = {};
+        for (const sName of scorerNames) {
+          const meanScore = trialResults.reduce((sum, t) => sum + t.scores[sName].score, 0) / trials;
+          finalScores[sName] = {
+            score: meanScore,
+            reason: trialResults[trialResults.length - 1].scores[sName]?.reason
+          };
+        }
+      } else {
+        finalResult = await wrapTask(task, input, timeout);
+        if (finalResult.error) {
+          finalScores = failureScores(scorerNames, finalResult.error);
+        } else {
+          finalScores = {};
+          for (const [sName, scorer] of Object.entries(scorers)) {
+            const sr = await scorer({
+              input,
+              output: finalResult.output,
+              expected: input.expected
+            });
+            finalScores[sName] = {
+              score: clampScore(sr.score, sName),
+              reason: sr.reason
+            };
+          }
+        }
+      }
+      const caseId = crypto.randomUUID();
+      const caseData = {
+        id: caseId,
+        run_id: runId,
+        idx: index,
+        input,
+        output: finalResult.output || null,
+        expected: input.expected,
+        latency_ms: finalResult.latencyMs,
+        tokens_in: finalResult.tokensIn,
+        tokens_out: finalResult.tokensOut,
+        error: finalResult.error ? serializeError(finalResult.error) : void 0
+      };
+      store.saveCases([caseData]);
+      const scoreDataList = scorerNames.map((sName) => ({
+        id: crypto.randomUUID(),
+        case_id: caseId,
+        scorer_name: sName,
+        score: finalScores[sName].score,
+        reason: finalScores[sName].reason
+      }));
+      store.saveScores(scoreDataList);
+      allCaseScores.push({
+        index,
+        scores: Object.fromEntries(
+          scorerNames.map((sName) => [sName, finalScores[sName].score])
+        ),
+        latencyMs: finalResult.latencyMs,
+        tokensIn: finalResult.tokensIn,
+        tokensOut: finalResult.tokensOut
+      });
+      if (finalResult.error) {
+        emitter.emit("case:error", {
+          runId,
+          index,
+          error: errorMessage(finalResult.error)
+        });
+      }
+      emitter.emit("case:scored", {
+        runId,
+        index,
+        input,
+        output: finalResult.output,
+        expected: input.expected,
+        scores: finalScores,
+        error: finalResult.error,
+        latencyMs: finalResult.latencyMs,
+        tokensIn: finalResult.tokensIn,
+        tokensOut: finalResult.tokensOut
+      });
+    } finally {
+      semaphore.release();
+    }
+  };
+  const batches = batchSize ? Array.from(
+    { length: Math.ceil(items.length / batchSize) },
+    (_, i) => items.slice(i * batchSize, (i + 1) * batchSize)
+  ) : [items];
+  try {
+    for (const batch of batches) {
+      await Promise.all(batch.map(processItem));
+    }
+  } catch (err) {
+    store.finishRun(runId, "failed");
+    throw err;
+  }
+  const summary = computeSummary(allCaseScores, scorerNames, threshold);
+  store.finishRun(runId, "completed", summary);
+  emitter.emit("run:end", { runId, summary });
+  return summary;
+}
+function computeSummary(cases, scorerNames, threshold) {
+  const totalCases = cases.length;
+  let passCount = 0;
+  let failCount = 0;
+  let totalLatencyMs = 0;
+  let totalTokensIn = 0;
+  let totalTokensOut = 0;
+  const scoreSums = {};
+  for (const name of scorerNames) {
+    scoreSums[name] = 0;
+  }
+  for (const c of cases) {
+    totalLatencyMs += c.latencyMs;
+    totalTokensIn += c.tokensIn;
+    totalTokensOut += c.tokensOut;
+    let allPass = true;
+    for (const name of scorerNames) {
+      const score = c.scores[name] ?? 0;
+      scoreSums[name] += score;
+      if (score < threshold) allPass = false;
+    }
+    if (allPass) passCount++;
+    else failCount++;
+  }
+  const meanScores = {};
+  for (const name of scorerNames) {
+    meanScores[name] = totalCases > 0 ? scoreSums[name] / totalCases : 0;
+  }
+  return {
+    totalCases,
+    passCount,
+    failCount,
+    meanScores,
+    totalLatencyMs,
+    totalTokensIn,
+    totalTokensOut
+  };
+}
+
+// packages/evals/src/comparison/index.ts
+function categorize(delta, tolerance) {
+  if (Math.abs(delta) <= tolerance) return "unchanged";
+  return delta > 0 ? "improved" : "regressed";
+}
+function buildScoreMap(cases) {
+  const map = /* @__PURE__ */ new Map();
+  for (const c of cases) {
+    const scores = {};
+    for (const s of c.scores) {
+      scores[s.scorer_name] = s.score;
+    }
+    map.set(c.idx, scores);
+  }
+  return map;
+}
+function getAllCasesWithScores(store, runId) {
+  const cases = store.getCases(runId);
+  const withScores = store.getFailingCases(runId, Infinity);
+  const scoredMap = new Map(withScores.map((c) => [c.id, c]));
+  return cases.map((c) => scoredMap.get(c.id) ?? { ...c, scores: [] });
+}
+function compareRuns(store, baselineRunId, candidateRunId, options) {
+  const tolerance = options?.tolerance ?? 0.01;
+  const regressionThreshold = options?.regressionThreshold ?? 0.05;
+  const baselineCases = getAllCasesWithScores(store, baselineRunId);
+  const candidateCases = getAllCasesWithScores(store, candidateRunId);
+  if (baselineCases.length !== candidateCases.length) {
+    console.warn(
+      `Run case count mismatch: baseline=${baselineCases.length}, candidate=${candidateCases.length}. Comparing intersection only.`
+    );
+  }
+  const baselineMap = buildScoreMap(baselineCases);
+  const candidateMap = buildScoreMap(candidateCases);
+  const allScorerNames = /* @__PURE__ */ new Set();
+  for (const scores of baselineMap.values()) {
+    for (const name of Object.keys(scores)) allScorerNames.add(name);
+  }
+  for (const scores of candidateMap.values()) {
+    for (const name of Object.keys(scores)) allScorerNames.add(name);
+  }
+  const commonIndices = [...baselineMap.keys()].filter(
+    (idx) => candidateMap.has(idx)
+  );
+  commonIndices.sort((a, b) => a - b);
+  const caseDiffs = [];
+  const scorerDeltas = {};
+  const scorerCounts = {};
+  for (const name of allScorerNames) {
+    scorerDeltas[name] = [];
+    scorerCounts[name] = { improved: 0, regressed: 0, unchanged: 0 };
+  }
+  for (const idx of commonIndices) {
+    const baseScores = baselineMap.get(idx);
+    const candScores = candidateMap.get(idx);
+    const diff = { index: idx, scorerDeltas: {} };
+    for (const name of allScorerNames) {
+      const baseline = baseScores[name] ?? 0;
+      const candidate = candScores[name] ?? 0;
+      const delta = candidate - baseline;
+      const change = categorize(delta, tolerance);
+      diff.scorerDeltas[name] = { baseline, candidate, delta, change };
+      scorerDeltas[name].push(delta);
+      if (change === "improved") scorerCounts[name].improved++;
+      else if (change === "regressed") scorerCounts[name].regressed++;
+      else scorerCounts[name].unchanged++;
+    }
+    caseDiffs.push(diff);
+  }
+  const scorerSummaries = {};
+  for (const name of allScorerNames) {
+    const deltas = scorerDeltas[name];
+    const meanDelta = deltas.length > 0 ? deltas.reduce((a, b) => a + b, 0) / deltas.length : 0;
+    scorerSummaries[name] = {
+      meanDelta,
+      improvedCount: scorerCounts[name].improved,
+      regressedCount: scorerCounts[name].regressed,
+      unchangedCount: scorerCounts[name].unchanged
+    };
+  }
+  const baselineSummary = store.getRunSummary(baselineRunId);
+  const candidateSummary = store.getRunSummary(candidateRunId);
+  const costDelta = {
+    latencyDeltaMs: candidateSummary.totalLatencyMs - baselineSummary.totalLatencyMs,
+    tokenInDelta: candidateSummary.totalTokensIn - baselineSummary.totalTokensIn,
+    tokenOutDelta: candidateSummary.totalTokensOut - baselineSummary.totalTokensOut
+  };
+  const regressionDetails = {};
+  let anyRegressed = false;
+  for (const [name, summary] of Object.entries(scorerSummaries)) {
+    const exceeds = summary.meanDelta < -regressionThreshold;
+    regressionDetails[name] = { meanDelta: summary.meanDelta, exceeds };
+    if (exceeds) anyRegressed = true;
+  }
+  return {
+    caseDiffs,
+    scorerSummaries,
+    costDelta,
+    totalCasesCompared: commonIndices.length,
+    regression: { regressed: anyRegressed, details: regressionDetails }
+  };
+}
+
+// packages/evals/src/reporters/console.ts
+import chalk from "chalk";
+
+// packages/evals/src/reporters/format.ts
+function formatDuration(ms) {
+  if (ms < 1e3) return `${ms}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+function formatTokens(n) {
+  if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
+  if (n >= 1e3) return `${(n / 1e3).toFixed(0)}k`;
+  return String(n);
+}
+function generateFilename(name, runId, ext) {
+  const slug = name.replace(/[^a-zA-Z0-9-_]/g, "-").toLowerCase();
+  const prefix = runId.slice(0, 8);
+  return `${slug}-${prefix}.${ext}`;
+}
+function stringifyUnknown(value, options) {
+  if (typeof value === "string") return value;
+  const space = options?.space ?? 0;
+  const fallback = options?.fallback ?? "null";
+  try {
+    return JSON.stringify(value, null, space) ?? fallback;
+  } catch {
+    return String(value);
+  }
+}
+function formatInputValue(value) {
+  return stringifyUnknown(value, { space: 0, fallback: "" });
+}
+function formatErrorValue(value) {
+  if (value == null) return "";
+  if (typeof value === "string") return value;
+  return stringifyUnknown(value, { space: 2, fallback: "" });
+}
+function escapeCsv(value) {
+  const str = stringifyUnknown(value, { space: 0, fallback: "null" });
+  if (str.includes(",") || str.includes('"') || str.includes("\n")) {
+    return `"${str.replace(/"/g, '""')}"`;
+  }
+  return str;
+}
+
+// packages/evals/src/reporters/shared.ts
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+var DEFAULT_OUTPUT_DIR = ".evals/reports";
+function resolveOutputDir(outputDir) {
+  return outputDir ?? DEFAULT_OUTPUT_DIR;
+}
+function getReportPath(outputDir, name, runId, ext) {
+  return join(outputDir, generateFilename(name, runId, ext));
+}
+async function writeRunReportFile(outputDir, name, runId, ext, content) {
+  await mkdir(outputDir, { recursive: true });
+  await writeFile(getReportPath(outputDir, name, runId, ext), content, "utf-8");
+}
+function getCaseStatus(result, threshold) {
+  if (result.error) return "error";
+  const passed = Object.values(result.scores).every(
+    (s) => s.score >= threshold
+  );
+  return passed ? "pass" : "fail";
+}
+function createRunEndFileReporter(options) {
+  const outputDir = resolveOutputDir(options.outputDir);
+  return {
+    async onRunEnd(data) {
+      const content = await options.render(data);
+      await writeRunReportFile(
+        outputDir,
+        data.name,
+        data.runId,
+        options.ext,
+        content
+      );
+    }
+  };
+}
+
+// packages/evals/src/reporters/console.ts
+function consoleReporter(options) {
+  const verbosity = options?.verbosity ?? "normal";
+  let totalCases = 0;
+  let completed = 0;
+  return {
+    onRunStart(data) {
+      totalCases = data.totalCases;
+      completed = 0;
+    },
+    onCaseEnd() {
+      completed++;
+      if (verbosity !== "quiet") {
+        process.stdout.write(
+          `\r  ${chalk.dim(`[${completed}/${totalCases}]`)}`
+        );
+      }
+    },
+    onRunEnd(data) {
+      if (verbosity !== "quiet") {
+        process.stdout.write("\r" + " ".repeat(30) + "\r");
+      }
+      renderSummaryTable(data);
+      if (verbosity === "quiet") return;
+      const sorted = [...data.cases].sort((a, b) => a.index - b.index);
+      if (verbosity === "verbose") {
+        for (const c of sorted) {
+          renderCaseDetail(c, data.threshold, {
+            includeIO: true,
+            maxStringLength: 2e4
+          });
+        }
+      } else {
+        const failing = sorted.filter(
+          (c) => getCaseStatus(c, data.threshold) !== "pass"
+        );
+        if (failing.length > 0) {
+          console.log(chalk.dim(`  Failing cases (${failing.length}):`));
+          console.log("");
+          for (const c of failing) {
+            renderCaseDetail(c, data.threshold, {
+              includeIO: true,
+              maxStringLength: 4e3
+            });
+          }
+        }
+      }
+    }
+  };
+}
+function indentBlock(text, spaces) {
+  const pad = " ".repeat(spaces);
+  return text.replace(/\r\n/g, "\n").split("\n").map((line) => pad + line).join("\n");
+}
+function truncateString(text, maxLength) {
+  if (text.length <= maxLength) return text;
+  return text.slice(0, maxLength) + "\u2026";
+}
+function renderSummaryTable(data) {
+  const { summary } = data;
+  const scoreStr = Object.entries(summary.meanScores).map(([name, score]) => `${name}: ${score.toFixed(3)}`).join(", ");
+  console.log("");
+  console.log(chalk.bold("  Summary"));
+  console.log(chalk.dim("  " + "\u2500".repeat(60)));
+  console.log(`  ${chalk.dim("Eval:")}     ${data.name}`);
+  console.log(`  ${chalk.dim("Model:")}    ${data.model}`);
+  console.log(`  ${chalk.dim("Cases:")}    ${summary.totalCases}`);
+  console.log(
+    `  ${chalk.dim("Pass/Fail:")} ${chalk.green(String(summary.passCount))} / ${chalk.red(String(summary.failCount))}`
+  );
+  console.log(`  ${chalk.dim("Scores:")}   ${scoreStr}`);
+  console.log(
+    `  ${chalk.dim("Duration:")} ${formatDuration(summary.totalLatencyMs)}`
+  );
+  console.log(
+    `  ${chalk.dim("Tokens:")}   ${formatTokens(summary.totalTokensIn + summary.totalTokensOut)}`
+  );
+  console.log(chalk.dim("  " + "\u2500".repeat(60)));
+  console.log("");
+}
+function renderCaseDetail(c, threshold, options) {
+  const entries = Object.entries(c.scores);
+  const failed = entries.some(([, s]) => s.score < threshold);
+  const prefix = failed ? chalk.red("FAIL") : chalk.green("PASS");
+  const includeIO = options?.includeIO ?? false;
+  const maxStringLength = options?.maxStringLength ?? 4e3;
+  console.log(`  ${prefix} ${chalk.dim(`Case #${c.index}`)}`);
+  const inputStr = stringifyUnknown(c.input, {
+    space: 2,
+    fallback: String(c.input)
+  });
+  console.log(`    ${chalk.dim("Input:")}  ${inputStr}`);
+  if (includeIO) {
+    console.log(`    ${chalk.dim("Output:")}`);
+    console.log(indentBlock(truncateString(c.output, maxStringLength), 6));
+    console.log(`    ${chalk.dim("Expected:")}`);
+    const expectedStrRaw = stringifyUnknown(c.expected, {
+      space: 2,
+      fallback: String(c.expected)
+    });
+    console.log(
+      indentBlock(truncateString(expectedStrRaw, maxStringLength), 6)
+    );
+  }
+  for (const [name, s] of entries) {
+    const scoreColor = s.score >= threshold ? chalk.green : chalk.red;
+    const reasonStr = s.reason ? ` \u2014 ${s.reason}` : "";
+    console.log(
+      `    ${chalk.dim(name + ":")} ${scoreColor(s.score.toFixed(3))}${reasonStr}`
+    );
+  }
+  if (c.error) {
+    console.log(`    ${chalk.dim("Error:")}`);
+    const errorStr = formatErrorValue(c.error);
+    console.log(`      ${chalk.red(errorStr)}`);
+  }
+  console.log("");
+}
+
+// packages/evals/src/reporters/json.ts
+import { appendFile, mkdir as mkdir2 } from "node:fs/promises";
+function jsonReporter(options) {
+  const outputDir = resolveOutputDir(options?.outputDir);
+  const pretty = options?.pretty ?? true;
+  let streamFilename = "";
+  return {
+    async onRunStart(data) {
+      await mkdir2(outputDir, { recursive: true });
+      streamFilename = getReportPath(outputDir, data.name, data.runId, "jsonl");
+    },
+    async onCaseEnd(data) {
+      const line = stringifyUnknown(data, { space: 0, fallback: "null" });
+      await appendFile(streamFilename, line + "\n", "utf-8");
+    },
+    async onRunEnd(data) {
+      const content = stringifyUnknown(data, {
+        space: pretty ? 2 : 0,
+        fallback: "null"
+      });
+      await writeRunReportFile(
+        outputDir,
+        data.name,
+        data.runId,
+        "json",
+        content
+      );
+    }
+  };
+}
+
+// packages/evals/src/reporters/csv.ts
+function csvReporter(options) {
+  return createRunEndFileReporter({
+    outputDir: options?.outputDir,
+    ext: "csv",
+    render(data) {
+      const scorerNames = Object.keys(data.summary.meanScores);
+      const headerParts = [
+        "index",
+        "input",
+        "output",
+        "expected",
+        "error",
+        "latency_ms",
+        "tokens_in",
+        "tokens_out"
+      ];
+      for (const name of scorerNames) {
+        headerParts.push(`${name}_score`, `${name}_reason`);
+      }
+      const rows = [headerParts.join(",")];
+      for (const c of data.cases) {
+        const parts = [
+          String(c.index),
+          escapeCsv(c.input),
+          escapeCsv(c.output),
+          escapeCsv(c.expected),
+          escapeCsv(c.error ?? ""),
+          String(c.latencyMs),
+          String(c.tokensIn),
+          String(c.tokensOut)
+        ];
+        for (const name of scorerNames) {
+          const s = c.scores[name];
+          parts.push(String(s?.score ?? ""), escapeCsv(s?.reason ?? ""));
+        }
+        rows.push(parts.join(","));
+      }
+      return rows.join("\n") + "\n";
+    }
+  });
+}
+
+// packages/evals/src/reporters/markdown.ts
+function markdownReporter(options) {
+  return createRunEndFileReporter({
+    outputDir: options?.outputDir,
+    ext: "md",
+    render(data) {
+      const { summary } = data;
+      const scorerNames = Object.keys(summary.meanScores);
+      const lines = [];
+      lines.push(`# ${data.name}`);
+      lines.push("");
+      lines.push(`**Model:** ${data.model}`);
+      lines.push(
+        `**Cases:** ${summary.totalCases} (${summary.passCount} pass, ${summary.failCount} fail)`
+      );
+      lines.push(`**Duration:** ${formatDuration(summary.totalLatencyMs)}`);
+      lines.push(
+        `**Tokens:** ${formatTokens(summary.totalTokensIn + summary.totalTokensOut)}`
+      );
+      lines.push("");
+      lines.push("## Scores");
+      lines.push("");
+      lines.push("| Scorer | Mean |");
+      lines.push("|--------|------|");
+      for (const [name, score] of Object.entries(summary.meanScores)) {
+        lines.push(`| ${name} | ${score.toFixed(3)} |`);
+      }
+      lines.push("");
+      lines.push("## Cases");
+      lines.push("");
+      const caseHeader = [
+        "#",
+        "Status",
+        "Input",
+        ...scorerNames,
+        "Latency",
+        "Error"
+      ];
+      lines.push(`| ${caseHeader.join(" | ")} |`);
+      lines.push(`| ${caseHeader.map(() => "---").join(" | ")} |`);
+      for (const c of data.cases) {
+        const statusValue = getCaseStatus(c, data.threshold);
+        const status = statusValue === "error" ? "\u{1F534} Error" : statusValue === "pass" ? "\u2705 Pass" : "\u274C Fail";
+        const input = formatInputValue(c.input).slice(0, 60);
+        const scores = scorerNames.map(
+          (name) => c.scores[name]?.score.toFixed(3) ?? "-"
+        );
+        const error = c.error ? formatErrorValue(c.error).replace(/\r?\n/g, "<br>").replace(/\|/g, "\\|") : "-";
+        const row = [
+          String(c.index),
+          status,
+          input,
+          ...scores,
+          `${c.latencyMs}ms`,
+          error
+        ];
+        lines.push(`| ${row.join(" | ")} |`);
+      }
+      lines.push("");
+      return lines.join("\n");
+    }
+  });
+}
+
+// packages/evals/src/reporters/html.ts
+function htmlReporter(options) {
+  return createRunEndFileReporter({
+    outputDir: options?.outputDir,
+    ext: "html",
+    render: renderHtml
+  });
+}
+function esc(str) {
+  return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function renderHtml(data) {
+  const { summary } = data;
+  const scorerNames = Object.keys(summary.meanScores);
+  const caseRows = data.cases.map((c) => {
+    const status = getCaseStatus(c, data.threshold);
+    const statusLabel = status === "error" ? "ERROR" : status === "pass" ? "PASS" : "FAIL";
+    const scoresCells = scorerNames.map((name) => {
+      const s = c.scores[name];
+      const score = s?.score ?? 0;
+      const cls = score >= data.threshold ? "pass" : "fail";
+      const reason = s?.reason ? ` title="${esc(s.reason)}"` : "";
+      return `<td class="${cls}"${reason}>${score.toFixed(3)}</td>`;
+    }).join("");
+    return `<tr class="${status}">
+        <td>${c.index}</td>
+        <td class="${status}">${statusLabel}</td>
+        <td class="text">${esc(formatInputValue(c.input).slice(0, 120))}</td>
+        <td class="text">${esc(c.output.slice(0, 120))}</td>
+        ${scoresCells}
+        <td>${c.latencyMs}ms</td>
+        <td class="error-text">${c.error ? esc(formatErrorValue(c.error)) : ""}</td>
+      </tr>`;
+  }).join("\n");
+  const scorerHeaders = scorerNames.map((n) => `<th>${esc(n)}</th>`).join("");
+  const meanScoreRows = Object.entries(summary.meanScores).map(
+    ([name, score]) => `<tr><td>${esc(name)}</td><td>${score.toFixed(3)}</td></tr>`
+  ).join("");
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>${esc(data.name)} \u2014 Eval Report</title>
+<style>
+  * { box-sizing: border-box; margin: 0; padding: 0; }
+  body { font-family: system-ui, -apple-system, sans-serif; background: #f8f9fa; color: #1a1a1a; padding: 2rem; }
+  h1 { font-size: 1.5rem; margin-bottom: 0.5rem; }
+  .meta { color: #666; margin-bottom: 1.5rem; font-size: 0.9rem; }
+  .meta span { margin-right: 1.5rem; }
+  .summary-table, .cases-table { width: 100%; border-collapse: collapse; margin-bottom: 2rem; }
+  .summary-table th, .summary-table td,
+  .cases-table th, .cases-table td { padding: 0.5rem 0.75rem; border: 1px solid #ddd; text-align: left; font-size: 0.85rem; }
+  .summary-table th, .cases-table th { background: #f1f3f5; font-weight: 600; }
+  .cases-table .text { max-width: 300px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+  .cases-table .error-text { max-width: 480px; white-space: pre-wrap; word-break: break-word; }
+  .pass { color: #2b8a3e; }
+  .fail { color: #c92a2a; }
+  .error { color: #e67700; }
+  tr.pass:hover, tr.fail:hover, tr.error:hover { background: #f1f3f5; }
+  td.pass { background: #ebfbee; }
+  td.fail { background: #fff5f5; }
+  h2 { font-size: 1.2rem; margin: 1.5rem 0 0.75rem; }
+</style>
+</head>
+<body>
+  <h1>${esc(data.name)}</h1>
+  <div class="meta">
+    <span><strong>Model:</strong> ${esc(data.model)}</span>
+    <span><strong>Cases:</strong> ${summary.totalCases}</span>
+    <span><strong>Pass:</strong> ${summary.passCount}</span>
+    <span><strong>Fail:</strong> ${summary.failCount}</span>
+    <span><strong>Duration:</strong> ${formatDuration(summary.totalLatencyMs)}</span>
+    <span><strong>Tokens:</strong> ${formatTokens(summary.totalTokensIn + summary.totalTokensOut)}</span>
+  </div>
+
+  <h2>Mean Scores</h2>
+  <table class="summary-table">
+    <thead><tr><th>Scorer</th><th>Mean</th></tr></thead>
+    <tbody>${meanScoreRows}</tbody>
+  </table>
+
+  <h2>Cases</h2>
+  <table class="cases-table">
+    <thead>
+      <tr>
+        <th>#</th>
+        <th>Status</th>
+        <th>Input</th>
+        <th>Output</th>
+        ${scorerHeaders}
+        <th>Latency</th>
+        <th>Error</th>
+      </tr>
+    </thead>
+    <tbody>
+      ${caseRows}
+    </tbody>
+  </table>
+</body>
+</html>`;
+}
+
+// packages/evals/src/evaluate/index.ts
+async function evaluate(options) {
+  if ("models" in options) {
+    return evaluateEach(options);
+  }
+  return evaluateSingle(options);
+}
+function resolveStore(store) {
+  return store instanceof RunStore ? store : new RunStore(store);
+}
+function wireReporters(reporters) {
+  const emitter = new EvalEmitter();
+  const cases = [];
+  let runId = "";
+  emitter.on("run:start", (data) => {
+    runId = data.runId;
+    for (const r of reporters) r.onRunStart?.(data);
+  });
+  emitter.on("case:scored", (data) => {
+    const result = {
+      runId: data.runId,
+      index: data.index,
+      input: data.input,
+      output: data.output,
+      expected: data.expected,
+      scores: data.scores,
+      error: data.error ?? null,
+      latencyMs: data.latencyMs,
+      tokensIn: data.tokensIn,
+      tokensOut: data.tokensOut
+    };
+    cases.push(result);
+    for (const r of reporters) r.onCaseEnd?.(result);
+  });
+  return { emitter, cases, getRunId: () => runId };
+}
+async function notifyRunEnd(reporters, data) {
+  data.cases.sort((a, b) => a.index - b.index);
+  await Promise.all(reporters.map((r) => r.onRunEnd?.(data)));
+}
+async function evaluateSingle(options) {
+  const store = resolveStore(options.store);
+  const threshold = options.threshold ?? 0.5;
+  const { emitter, cases, getRunId } = wireReporters(options.reporters);
+  const summary = await runEval({
+    name: options.name,
+    model: options.model,
+    dataset: options.dataset,
+    task: options.task,
+    scorers: options.scorers,
+    store,
+    emitter,
+    suiteId: options.suiteId,
+    maxConcurrency: options.maxConcurrency,
+    timeout: options.timeout,
+    trials: options.trials,
+    threshold: options.threshold
+  });
+  await notifyRunEnd(options.reporters, {
+    runId: getRunId(),
+    name: options.name,
+    model: options.model,
+    summary,
+    cases,
+    threshold
+  });
+  return summary;
+}
+async function evaluateEach(options) {
+  const store = resolveStore(options.store);
+  const items = [];
+  for await (const item of options.dataset) {
+    items.push(item);
+  }
+  const suite = store.createSuite(options.name);
+  return Promise.all(
+    options.models.map(
+      (variant) => evaluateSingle({
+        name: `${options.name} [${variant.name}]`,
+        model: variant.name,
+        dataset: dataset(items),
+        task: (input) => options.task(input, variant),
+        scorers: options.scorers,
+        reporters: options.reporters,
+        store,
+        suiteId: suite.id,
+        maxConcurrency: options.maxConcurrency,
+        timeout: options.timeout,
+        trials: options.trials,
+        threshold: options.threshold
+      })
+    )
+  );
+}
+export {
+  Dataset,
+  EvalEmitter,
+  RunStore,
+  all,
+  any,
+  compareRuns,
+  consoleReporter,
+  csvReporter,
+  dataset,
+  evaluate,
+  exactMatch,
+  factuality,
+  hf,
+  htmlReporter,
+  includes,
+  jsonMatch,
+  jsonReporter,
+  levenshtein,
+  llmJudge,
+  markdownReporter,
+  regex,
+  runEval,
+  weighted
+};
+//# sourceMappingURL=index.js.map