@deepagents/evals 0.19.0

package/dist/evaluate/index.js

@@ -0,0 +1,977 @@
+// packages/evals/src/dataset/index.ts
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { extname } from "node:path";
+import { createInterface } from "node:readline";
+var Dataset = class _Dataset {
+  #source;
+  constructor(source) {
+    this.#source = source;
+  }
+  map(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        yield fn(item);
+      }
+    });
+  }
+  filter(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        if (fn(item)) yield item;
+      }
+    });
+  }
+  limit(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      let count = 0;
+      for await (const item of source()) {
+        if (count >= n) return;
+        yield item;
+        count++;
+      }
+    });
+  }
+  shuffle() {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      for (let i = items.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      yield* items;
+    });
+  }
+  sample(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      const count = Math.min(Math.max(0, n), items.length);
+      for (let i = items.length - 1; i > items.length - count - 1; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      for (let i = items.length - count; i < items.length; i++) {
+        yield items[i];
+      }
+    });
+  }
+  async toArray() {
+    const result = [];
+    for await (const item of this.#source()) {
+      result.push(item);
+    }
+    return result;
+  }
+  [Symbol.asyncIterator]() {
+    return this.#source()[Symbol.asyncIterator]();
+  }
+};
+function parseCSVLine(line) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += char;
+      }
+    } else {
+      if (char === '"' && current === "") {
+        inQuotes = true;
+      } else if (char === ",") {
+        fields.push(current);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+  }
+  fields.push(current);
+  return fields;
+}
+function loadJSON(filePath) {
+  return async function* () {
+    const content = await readFile(filePath, "utf-8");
+    const data = JSON.parse(content);
+    if (!Array.isArray(data)) {
+      throw new Error(`JSON file "${filePath}" does not contain an array`);
+    }
+    yield* data;
+  };
+}
+function loadJSONL(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (trimmed) {
+          yield JSON.parse(trimmed);
+        }
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function loadCSV(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      let headers;
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        const fields = parseCSVLine(trimmed);
+        if (!headers) {
+          headers = fields;
+          continue;
+        }
+        const row = {};
+        for (let i = 0; i < headers.length; i++) {
+          row[headers[i]] = fields[i] ?? "";
+        }
+        yield row;
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function dataset(source) {
+  if (Array.isArray(source)) {
+    return new Dataset(async function* () {
+      yield* source;
+    });
+  }
+  if (typeof source === "object" && Symbol.asyncIterator in source) {
+    return new Dataset(() => source);
+  }
+  const ext = extname(source).toLowerCase();
+  switch (ext) {
+    case ".json":
+      return new Dataset(loadJSON(source));
+    case ".jsonl":
+      return new Dataset(loadJSONL(source));
+    case ".csv":
+      return new Dataset(loadCSV(source));
+    default:
+      throw new Error(
+        `Unsupported file extension "${ext}" for dataset file "${source}". Supported: .json, .jsonl, .csv`
+      );
+  }
+}
+
+// packages/evals/src/engine/index.ts
+import { EventEmitter } from "node:events";
+var EvalEmitter = class extends EventEmitter {
+  on(event, listener) {
+    return super.on(event, listener);
+  }
+  emit(event, data) {
+    return super.emit(event, data);
+  }
+};
+function errorMessage(err) {
+  if (err instanceof Error) {
+    return `${err.name}: ${err.message}`;
+  }
+  if (typeof err === "string") return err;
+  if (err == null) return "Unknown error";
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return String(err);
+  }
+}
+function serializeError(err) {
+  if (err instanceof Error) {
+    return JSON.stringify({
+      name: err.name,
+      message: err.message,
+      stack: err.stack,
+      cause: err.cause instanceof Error ? {
+        name: err.cause.name,
+        message: err.cause.message
+      } : err.cause
+    });
+  }
+  if (typeof err === "string") return JSON.stringify({ message: err });
+  if (err == null) return JSON.stringify({ message: "Unknown error" });
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return JSON.stringify({ message: String(err) });
+  }
+}
+function failureScores(scorerNames, error) {
+  const reason = `Task failed: ${errorMessage(error)}`;
+  const scores = {};
+  for (const scorerName of scorerNames) {
+    scores[scorerName] = { score: 0, reason };
+  }
+  return scores;
+}
+function createSemaphore(maxConcurrency) {
+  let active = 0;
+  const queue = [];
+  return {
+    async acquire() {
+      if (active < maxConcurrency) {
+        active++;
+        return;
+      }
+      return new Promise((resolve) => queue.push(resolve));
+    },
+    release() {
+      active--;
+      const next = queue.shift();
+      if (next) {
+        active++;
+        next();
+      }
+    }
+  };
+}
+async function wrapTask(task, input, timeoutMs) {
+  const start = performance.now();
+  let timerId;
+  try {
+    const result = await Promise.race([
+      task(input),
+      new Promise((_, reject) => {
+        timerId = setTimeout(
+          () => reject(new Error("timeout exceeded")),
+          timeoutMs
+        );
+      })
+    ]);
+    clearTimeout(timerId);
+    const latencyMs = Math.round(performance.now() - start);
+    return {
+      output: result.output,
+      latencyMs,
+      tokensIn: result.usage?.inputTokens ?? 0,
+      tokensOut: result.usage?.outputTokens ?? 0
+    };
+  } catch (err) {
+    clearTimeout(timerId);
+    const latencyMs = Math.round(performance.now() - start);
+    return {
+      output: "",
+      latencyMs,
+      tokensIn: 0,
+      tokensOut: 0,
+      error: err
+    };
+  }
+}
+function clampScore(score, scorerName) {
+  if (score < 0 || score > 1) {
+    console.warn(
+      `Scorer "${scorerName}" returned out-of-range score ${score}, clamping to 0..1`
+    );
+    return Math.max(0, Math.min(1, score));
+  }
+  return score;
+}
+async function runEval(config) {
+  const {
+    name,
+    model,
+    dataset: ds,
+    task,
+    scorers,
+    store,
+    suiteId,
+    maxConcurrency = 10,
+    batchSize,
+    timeout = 3e4,
+    trials = 1,
+    threshold = 0.5
+  } = config;
+  const emitter = config.emitter ?? new EvalEmitter();
+  const resolvedSuiteId = suiteId ?? store.createSuite(name).id;
+  const runId = store.createRun({
+    suite_id: resolvedSuiteId,
+    name,
+    model,
+    config: config.config
+  });
+  const items = [];
+  let idx = 0;
+  for await (const item of ds) {
+    items.push({ index: idx++, input: item });
+  }
+  emitter.emit("run:start", { runId, totalCases: items.length, name, model });
+  const semaphore = createSemaphore(maxConcurrency);
+  const scorerNames = Object.keys(scorers);
+  const allCaseScores = [];
+  const processItem = async ({ index, input }) => {
+    await semaphore.acquire();
+    try {
+      emitter.emit("case:start", { runId, index, input });
+      let finalResult;
+      let finalScores;
+      if (trials > 1) {
+        const trialResults = [];
+        for (let t = 0; t < trials; t++) {
+          const result = await wrapTask(task, input, timeout);
+          if (result.error) {
+            trialResults.push({
+              result,
+              scores: failureScores(scorerNames, result.error)
+            });
+          } else {
+            const scores = {};
+            for (const [sName, scorer] of Object.entries(scorers)) {
+              const sr = await scorer({
+                input,
+                output: result.output,
+                expected: input.expected
+              });
+              scores[sName] = {
+                score: clampScore(sr.score, sName),
+                reason: sr.reason
+              };
+            }
+            trialResults.push({ result, scores });
+          }
+        }
+        const lastSuccessful = [...trialResults].reverse().find((t) => !t.result.error);
+        const baseResult = lastSuccessful?.result ?? trialResults[trialResults.length - 1].result;
+        finalResult = {
+          output: baseResult.output,
+          latencyMs: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.latencyMs, 0) / trials
+          ),
+          tokensIn: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.tokensIn, 0) / trials
+          ),
+          tokensOut: Math.round(
+            trialResults.reduce((sum, t) => sum + t.result.tokensOut, 0) / trials
+          ),
+          error: lastSuccessful ? void 0 : baseResult.error
+        };
+        finalScores = {};
+        for (const sName of scorerNames) {
+          const meanScore = trialResults.reduce((sum, t) => sum + t.scores[sName].score, 0) / trials;
+          finalScores[sName] = {
+            score: meanScore,
+            reason: trialResults[trialResults.length - 1].scores[sName]?.reason
+          };
+        }
+      } else {
+        finalResult = await wrapTask(task, input, timeout);
+        if (finalResult.error) {
+          finalScores = failureScores(scorerNames, finalResult.error);
+        } else {
+          finalScores = {};
+          for (const [sName, scorer] of Object.entries(scorers)) {
+            const sr = await scorer({
+              input,
+              output: finalResult.output,
+              expected: input.expected
+            });
+            finalScores[sName] = {
+              score: clampScore(sr.score, sName),
+              reason: sr.reason
+            };
+          }
+        }
+      }
+      const caseId = crypto.randomUUID();
+      const caseData = {
+        id: caseId,
+        run_id: runId,
+        idx: index,
+        input,
+        output: finalResult.output || null,
+        expected: input.expected,
+        latency_ms: finalResult.latencyMs,
+        tokens_in: finalResult.tokensIn,
+        tokens_out: finalResult.tokensOut,
+        error: finalResult.error ? serializeError(finalResult.error) : void 0
+      };
+      store.saveCases([caseData]);
+      const scoreDataList = scorerNames.map((sName) => ({
+        id: crypto.randomUUID(),
+        case_id: caseId,
+        scorer_name: sName,
+        score: finalScores[sName].score,
+        reason: finalScores[sName].reason
+      }));
+      store.saveScores(scoreDataList);
+      allCaseScores.push({
+        index,
+        scores: Object.fromEntries(
+          scorerNames.map((sName) => [sName, finalScores[sName].score])
+        ),
+        latencyMs: finalResult.latencyMs,
+        tokensIn: finalResult.tokensIn,
+        tokensOut: finalResult.tokensOut
+      });
+      if (finalResult.error) {
+        emitter.emit("case:error", {
+          runId,
+          index,
+          error: errorMessage(finalResult.error)
+        });
+      }
+      emitter.emit("case:scored", {
+        runId,
+        index,
+        input,
+        output: finalResult.output,
+        expected: input.expected,
+        scores: finalScores,
+        error: finalResult.error,
+        latencyMs: finalResult.latencyMs,
+        tokensIn: finalResult.tokensIn,
+        tokensOut: finalResult.tokensOut
+      });
+    } finally {
+      semaphore.release();
+    }
+  };
+  const batches = batchSize ? Array.from(
+    { length: Math.ceil(items.length / batchSize) },
+    (_, i) => items.slice(i * batchSize, (i + 1) * batchSize)
+  ) : [items];
+  try {
+    for (const batch of batches) {
+      await Promise.all(batch.map(processItem));
+    }
+  } catch (err) {
+    store.finishRun(runId, "failed");
+    throw err;
+  }
+  const summary = computeSummary(allCaseScores, scorerNames, threshold);
+  store.finishRun(runId, "completed", summary);
+  emitter.emit("run:end", { runId, summary });
+  return summary;
+}
+function computeSummary(cases, scorerNames, threshold) {
+  const totalCases = cases.length;
+  let passCount = 0;
+  let failCount = 0;
+  let totalLatencyMs = 0;
+  let totalTokensIn = 0;
+  let totalTokensOut = 0;
+  const scoreSums = {};
+  for (const name of scorerNames) {
+    scoreSums[name] = 0;
+  }
+  for (const c of cases) {
+    totalLatencyMs += c.latencyMs;
+    totalTokensIn += c.tokensIn;
+    totalTokensOut += c.tokensOut;
+    let allPass = true;
+    for (const name of scorerNames) {
+      const score = c.scores[name] ?? 0;
+      scoreSums[name] += score;
+      if (score < threshold) allPass = false;
+    }
+    if (allPass) passCount++;
+    else failCount++;
+  }
+  const meanScores = {};
+  for (const name of scorerNames) {
+    meanScores[name] = totalCases > 0 ? scoreSums[name] / totalCases : 0;
+  }
+  return {
+    totalCases,
+    passCount,
+    failCount,
+    meanScores,
+    totalLatencyMs,
+    totalTokensIn,
+    totalTokensOut
+  };
+}
+
+// packages/evals/src/store/index.ts
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+// packages/evals/src/store/ddl.sqlite.sql
+var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
+
+// packages/evals/src/store/index.ts
+var RunStore = class {
+  #db;
+  #statements = /* @__PURE__ */ new Map();
+  #stmt(sql) {
+    let stmt = this.#statements.get(sql);
+    if (!stmt) {
+      stmt = this.#db.prepare(sql);
+      this.#statements.set(sql, stmt);
+    }
+    return stmt;
+  }
+  #transaction(fn) {
+    this.#db.exec("BEGIN TRANSACTION");
+    try {
+      const result = fn();
+      this.#db.exec("COMMIT");
+      return result;
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+  constructor(pathOrDb) {
+    if (pathOrDb instanceof DatabaseSync) {
+      this.#db = pathOrDb;
+    } else {
+      const dbPath = pathOrDb ?? ".evals/store.db";
+      mkdirSync(dirname(dbPath), { recursive: true });
+      this.#db = new DatabaseSync(dbPath);
+    }
+    this.#db.exec(ddl_sqlite_default);
+    this.#migrateRunsTableToSuiteRequired();
+    this.#migratePromptsTableIfNeeded();
+    this.#db.exec(
+      "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+    );
+  }
+  #migratePromptsTableIfNeeded() {
+    const columns = this.#stmt("PRAGMA table_info(prompts)").all();
+    if (columns.length === 0) return;
+    if (columns.some((column) => column.name === "version")) return;
+    this.#transaction(() => {
+      this.#db.exec("ALTER TABLE prompts RENAME TO prompts_legacy");
+      this.#db.exec(`
+        CREATE TABLE prompts (
+          id TEXT PRIMARY KEY,
+          name TEXT NOT NULL,
+          version INTEGER NOT NULL,
+          content TEXT NOT NULL,
+          created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+          UNIQUE(name, version)
+        )
+      `);
+      this.#db.exec(`
+        INSERT INTO prompts (id, name, version, content, created_at)
+        SELECT id, name, 1, content, created_at
+        FROM prompts_legacy
+      `);
+      this.#db.exec("DROP TABLE prompts_legacy");
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at)"
+      );
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+      );
+    });
+  }
+  #migrateRunsTableToSuiteRequired() {
+    const runColumns = this.#stmt("PRAGMA table_info(runs)").all();
+    if (runColumns.length === 0) return;
+    const suiteColumn = runColumns.find((column) => column.name === "suite_id");
+    const hasNonNullSuite = suiteColumn?.notnull === 1;
+    const runForeignKeys = this.#stmt(
+      "PRAGMA foreign_key_list(runs)"
+    ).all();
+    const suiteForeignKey = runForeignKeys.find(
+      (fk) => fk.from === "suite_id" && fk.table === "suites"
+    );
+    const hasCascadeDelete = suiteForeignKey?.on_delete === "CASCADE";
+    if (hasNonNullSuite && hasCascadeDelete) return;
+    this.#statements.clear();
+    this.#transaction(() => {
+      this.#db.exec(`
+        CREATE TABLE runs_next (
+          id TEXT PRIMARY KEY,
+          suite_id TEXT NOT NULL,
+          name TEXT NOT NULL,
+          model TEXT NOT NULL,
+          config TEXT,
+          started_at INTEGER NOT NULL,
+          finished_at INTEGER,
+          status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),
+          summary TEXT,
+          FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE
+        )
+      `);
+      this.#db.exec("DELETE FROM runs WHERE suite_id IS NULL");
+      this.#db.exec(`
+        INSERT INTO runs_next (id, suite_id, name, model, config, started_at, finished_at, status, summary)
+        SELECT r.id, r.suite_id, r.name, r.model, r.config, r.started_at, r.finished_at, r.status, r.summary
+        FROM runs r
+        JOIN suites s ON s.id = r.suite_id
+      `);
+      this.#db.exec("DROP TABLE runs");
+      this.#db.exec("ALTER TABLE runs_next RENAME TO runs");
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id)"
+      );
+      this.#db.exec(
+        "CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at)"
+      );
+    });
+    this.#statements.clear();
+  }
+  createSuite(name) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    this.#stmt(
+      "INSERT INTO suites (id, name, created_at) VALUES (?, ?, ?)"
+    ).run(id, name, now);
+    return { id, name, created_at: now };
+  }
+  createRun(run) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    this.#stmt(
+      "INSERT INTO runs (id, suite_id, name, model, config, started_at) VALUES (?, ?, ?, ?, ?, ?)"
+    ).run(
+      id,
+      run.suite_id,
+      run.name,
+      run.model,
+      run.config ? JSON.stringify(run.config) : null,
+      now
+    );
+    return id;
+  }
+  finishRun(runId, status, summary) {
+    this.#stmt(
+      "UPDATE runs SET finished_at = ?, status = ?, summary = ? WHERE id = ?"
+    ).run(Date.now(), status, summary ? JSON.stringify(summary) : null, runId);
+  }
+  saveCases(cases) {
+    this.#transaction(() => {
+      const stmt = this.#stmt(
+        "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+      );
+      for (const c of cases) {
+        stmt.run(
+          c.id,
+          c.run_id,
+          c.idx,
+          JSON.stringify(c.input),
+          c.output,
+          c.expected != null ? JSON.stringify(c.expected) : null,
+          c.latency_ms,
+          c.tokens_in,
+          c.tokens_out,
+          c.error ?? null
+        );
+      }
+    });
+  }
+  saveScores(scores) {
+    this.#transaction(() => {
+      const stmt = this.#stmt(
+        "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
+      );
+      for (const s of scores) {
+        stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
+      }
+    });
+  }
+  getRun(runId) {
+    const row = this.#stmt("SELECT * FROM runs WHERE id = ?").get(runId);
+    if (!row) return void 0;
+    return {
+      id: row.id,
+      suite_id: row.suite_id,
+      name: row.name,
+      model: row.model,
+      config: row.config ? JSON.parse(row.config) : null,
+      started_at: row.started_at,
+      finished_at: row.finished_at,
+      status: row.status,
+      summary: row.summary ? JSON.parse(row.summary) : null
+    };
+  }
+  listRuns(suiteId) {
+    const sql = suiteId ? "SELECT * FROM runs WHERE suite_id = ? ORDER BY started_at" : "SELECT * FROM runs ORDER BY started_at";
+    const rows = suiteId ? this.#stmt(sql).all(suiteId) : this.#stmt(sql).all();
+    return rows.map((row) => ({
+      id: row.id,
+      suite_id: row.suite_id,
+      name: row.name,
+      model: row.model,
+      config: row.config ? JSON.parse(row.config) : null,
+      started_at: row.started_at,
+      finished_at: row.finished_at,
+      status: row.status,
+      summary: row.summary ? JSON.parse(row.summary) : null
+    }));
+  }
+  getCases(runId) {
+    const rows = this.#stmt(
+      "SELECT * FROM cases WHERE run_id = ? ORDER BY idx"
+    ).all(runId);
+    return rows.map((row) => ({
+      id: row.id,
+      run_id: row.run_id,
+      idx: row.idx,
+      input: JSON.parse(row.input),
+      output: row.output,
+      expected: row.expected ? JSON.parse(row.expected) : null,
+      latency_ms: row.latency_ms,
+      tokens_in: row.tokens_in,
+      tokens_out: row.tokens_out,
+      error: row.error
+    }));
+  }
+  getFailingCases(runId, threshold = 0.5) {
+    const rows = this.#stmt(
+      `SELECT c.*, s.scorer_name, s.score, s.reason as score_reason
+       FROM cases c
+       JOIN scores s ON s.case_id = c.id
+       WHERE c.run_id = ? AND s.score < ?
+       ORDER BY c.idx`
+    ).all(runId, threshold);
+    const caseMap = /* @__PURE__ */ new Map();
+    for (const row of rows) {
+      let c = caseMap.get(row.id);
+      if (!c) {
+        c = {
+          id: row.id,
+          run_id: row.run_id,
+          idx: row.idx,
+          input: JSON.parse(row.input),
+          output: row.output,
+          expected: row.expected ? JSON.parse(row.expected) : null,
+          latency_ms: row.latency_ms,
+          tokens_in: row.tokens_in,
+          tokens_out: row.tokens_out,
+          error: row.error,
+          scores: []
+        };
+        caseMap.set(row.id, c);
+      }
+      c.scores.push({
+        scorer_name: row.scorer_name,
+        score: row.score,
+        reason: row.score_reason
+      });
+    }
+    return Array.from(caseMap.values());
+  }
+  getRunSummary(runId, threshold = 0.5) {
+    const totals = this.#stmt(
+      `SELECT
+        COUNT(DISTINCT c.id) as totalCases,
+        COALESCE(SUM(c.latency_ms), 0) as totalLatencyMs,
+        COALESCE(SUM(c.tokens_in), 0) as totalTokensIn,
+        COALESCE(SUM(c.tokens_out), 0) as totalTokensOut
+       FROM cases c WHERE c.run_id = ?`
+    ).get(runId);
+    const scorerMeans = this.#stmt(
+      `SELECT s.scorer_name, AVG(s.score) as meanScore
+       FROM scores s
+       JOIN cases c ON c.id = s.case_id
+       WHERE c.run_id = ?
+       GROUP BY s.scorer_name`
+    ).all(runId);
+    const meanScores = {};
+    for (const row of scorerMeans) {
+      meanScores[row.scorer_name] = row.meanScore;
+    }
+    const passFail = this.#stmt(
+      `SELECT c.id,
+        MIN(s.score) as minScore
+       FROM cases c
+       JOIN scores s ON s.case_id = c.id
+       WHERE c.run_id = ?
+       GROUP BY c.id`
+    ).all(runId);
+    let passCount = 0;
+    let failCount = 0;
+    for (const row of passFail) {
+      if (row.minScore >= threshold) passCount++;
+      else failCount++;
+    }
+    return {
+      totalCases: totals.totalCases,
+      passCount,
+      failCount,
+      meanScores,
+      totalLatencyMs: totals.totalLatencyMs,
+      totalTokensIn: totals.totalTokensIn,
+      totalTokensOut: totals.totalTokensOut
+    };
+  }
+  listSuites() {
+    const rows = this.#stmt(
+      "SELECT * FROM suites ORDER BY created_at DESC"
+    ).all();
+    return rows.map((row) => ({
+      id: row.id,
+      name: row.name,
+      created_at: row.created_at
+    }));
+  }
+  createPrompt(name, content) {
+    const id = crypto.randomUUID();
+    const now = Date.now();
+    const latest = this.#stmt(
+      "SELECT MAX(version) as latestVersion FROM prompts WHERE name = ?"
+    ).get(name);
+    const version = (latest?.latestVersion ?? 0) + 1;
+    this.#stmt(
+      "INSERT INTO prompts (id, name, version, content, created_at) VALUES (?, ?, ?, ?, ?)"
+    ).run(id, name, version, content, now);
+    return { id, name, version, content, created_at: now };
+  }
+  listPrompts() {
+    const rows = this.#stmt(
+      "SELECT * FROM prompts ORDER BY name COLLATE NOCASE ASC, version DESC"
+    ).all();
+    return rows.map((row) => ({
+      id: row.id,
+      name: row.name,
+      version: row.version,
+      content: row.content,
+      created_at: row.created_at
+    }));
+  }
+  getPrompt(id) {
+    const row = this.#stmt("SELECT * FROM prompts WHERE id = ?").get(id);
+    if (!row) return void 0;
+    return {
+      id: row.id,
+      name: row.name,
+      version: row.version,
+      content: row.content,
+      created_at: row.created_at
+    };
+  }
+  deletePrompt(id) {
+    this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
+  }
+};
+
+// packages/evals/src/evaluate/index.ts
+async function evaluate(options) {
+  if ("models" in options) {
+    return evaluateEach(options);
+  }
+  return evaluateSingle(options);
+}
+function resolveStore(store) {
+  return store instanceof RunStore ? store : new RunStore(store);
+}
+function wireReporters(reporters) {
+  const emitter = new EvalEmitter();
+  const cases = [];
+  let runId = "";
+  emitter.on("run:start", (data) => {
+    runId = data.runId;
+    for (const r of reporters) r.onRunStart?.(data);
+  });
+  emitter.on("case:scored", (data) => {
+    const result = {
+      runId: data.runId,
+      index: data.index,
+      input: data.input,
+      output: data.output,
+      expected: data.expected,
+      scores: data.scores,
+      error: data.error ?? null,
+      latencyMs: data.latencyMs,
+      tokensIn: data.tokensIn,
+      tokensOut: data.tokensOut
+    };
+    cases.push(result);
+    for (const r of reporters) r.onCaseEnd?.(result);
+  });
+  return { emitter, cases, getRunId: () => runId };
+}
+async function notifyRunEnd(reporters, data) {
+  data.cases.sort((a, b) => a.index - b.index);
+  await Promise.all(reporters.map((r) => r.onRunEnd?.(data)));
+}
+async function evaluateSingle(options) {
+  const store = resolveStore(options.store);
+  const threshold = options.threshold ?? 0.5;
+  const { emitter, cases, getRunId } = wireReporters(options.reporters);
+  const summary = await runEval({
+    name: options.name,
+    model: options.model,
+    dataset: options.dataset,
+    task: options.task,
+    scorers: options.scorers,
+    store,
+    emitter,
+    suiteId: options.suiteId,
+    maxConcurrency: options.maxConcurrency,
+    timeout: options.timeout,
+    trials: options.trials,
+    threshold: options.threshold
+  });
+  await notifyRunEnd(options.reporters, {
+    runId: getRunId(),
+    name: options.name,
+    model: options.model,
+    summary,
+    cases,
+    threshold
+  });
+  return summary;
+}
+async function evaluateEach(options) {
+  const store = resolveStore(options.store);
+  const items = [];
+  for await (const item of options.dataset) {
+    items.push(item);
+  }
+  const suite = store.createSuite(options.name);
+  return Promise.all(
+    options.models.map(
+      (variant) => evaluateSingle({
+        name: `${options.name} [${variant.name}]`,
+        model: variant.name,
+        dataset: dataset(items),
+        task: (input) => options.task(input, variant),
+        scorers: options.scorers,
+        reporters: options.reporters,
+        store,
+        suiteId: suite.id,
+        maxConcurrency: options.maxConcurrency,
+        timeout: options.timeout,
+        trials: options.trials,
+        threshold: options.threshold
+      })
+    )
+  );
+}
+export {
+  evaluate
+};
+//# sourceMappingURL=index.js.map