@deepagents/evals 7.0.2 → 7.1.1

package/dist/index.js

@@ -51,6 +51,15 @@ async function fetchPage(url) {
     );
   }
   const text = await response.text();
+  const page = parseJson(text, url);
+  if (!isHfRowsPage(page)) {
+    throw new Error(
+      `HuggingFace API returned an unexpected rows page from ${url}: ${text.slice(0, 200)}`
+    );
+  }
+  return page;
+}
+function parseJson(text, url) {
   try {
     return JSON.parse(text);
   } catch {
@@ -59,6 +68,14 @@ async function fetchPage(url) {
     );
   }
 }
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isHfRowsPage(value) {
+  return isRecord(value) && typeof value.num_rows_total === "number" && Array.isArray(value.rows) && value.rows.every(
+    (entry) => isRecord(entry) && typeof entry.row_idx === "number" && isRecord(entry.row)
+  );
+}
 
 // packages/evals/src/dataset/record-selection.ts
 function parsePositiveInt(token) {
@@ -250,6 +267,9 @@ function parseCSVLine(line) {
   fields.push(current);
   return fields;
 }
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 function loadJSON(filePath) {
   return async function* () {
     const content = await readFile(filePath, "utf-8");
@@ -257,7 +277,14 @@ function loadJSON(filePath) {
     if (!Array.isArray(data)) {
       throw new Error(`JSON file "${filePath}" does not contain an array`);
     }
-    yield* data;
+    for (const [index, row] of data.entries()) {
+      if (!isRecord2(row)) {
+        throw new Error(
+          `JSON file "${filePath}" has a non-object row at index ${index}`
+        );
+      }
+      yield row;
+    }
   };
 }
 function loadJSONL(filePath) {
@@ -269,9 +296,14 @@ function loadJSONL(filePath) {
     try {
       for await (const line of rl) {
         const trimmed = line.trim();
-        if (trimmed) {
-          yield JSON.parse(trimmed);
+        if (!trimmed) continue;
+        const row = JSON.parse(trimmed);
+        if (!isRecord2(row)) {
+          throw new Error(
+            `JSONL file "${filePath}" has a non-object row: ${trimmed.slice(0, 200)}`
+          );
         }
+        yield row;
       }
     } finally {
       rl.close();
@@ -410,6 +442,9 @@ var LevenshteinSimilarity = class {
     return previous[target.length];
   }
 };
+function isRecord3(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 function deepEqual(a, b) {
   if (a === b) return true;
   if (a == null || b == null) return false;
@@ -418,15 +453,13 @@ function deepEqual(a, b) {
     if (!Array.isArray(b) || a.length !== b.length) return false;
     return a.every((val, i) => deepEqual(val, b[i]));
   }
-  if (typeof a === "object") {
+  if (isRecord3(a)) {
+    if (!isRecord3(b)) return false;
     const keysA = Object.keys(a).sort();
     const keysB = Object.keys(b).sort();
     if (keysA.length !== keysB.length) return false;
     return keysA.every(
-      (key, i) => keysB[i] === key && deepEqual(
-        a[key],
-        b[key]
-      )
+      (key, i) => keysB[i] === key && deepEqual(a[key], b[key])
     );
   }
   return false;
@@ -544,6 +577,11 @@ var LlmJudge = class _LlmJudge {
       throw new Error(`Unexpected judge tool call: ${toolCall.function.name}`);
     }
     const payload = JSON.parse(toolCall.function.arguments);
+    if (!isRecord3(payload)) {
+      throw new Error(
+        `Judge sent select_choice arguments that are not an object: ${toolCall.function.arguments}`
+      );
+    }
     const choice = typeof payload.choice === "string" ? payload.choice.trim() : "";
     const score = this.#choices[choice];
     if (score === void 0) {
@@ -609,12 +647,135 @@ function weighted(config) {
 // packages/evals/src/store/index.ts
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
-import { DatabaseSync } from "node:sqlite";
+import {
+  DatabaseSync
+} from "node:sqlite";
 
 // packages/evals/src/store/ddl.sqlite.sql
 var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
 
 // packages/evals/src/store/index.ts
+var RUN_STATUSES = ["running", "completed", "failed"];
+function isRunStatus(value) {
+  return RUN_STATUSES.some((status) => status === value);
+}
+function matchesColumn(value, type) {
+  switch (type) {
+    case "text":
+      return typeof value === "string";
+    case "text?":
+      return value === null || typeof value === "string";
+    case "number":
+      return typeof value === "number";
+    case "number?":
+      return value === null || typeof value === "number";
+  }
+}
+function mismatchedColumns(row, columns) {
+  return Object.entries(columns).filter(([name, type]) => !matchesColumn(row[name], type)).map(([name]) => name);
+}
+function hasColumns(row, columns) {
+  return mismatchedColumns(row, columns).length === 0;
+}
+function readRow(row, columns, sql2) {
+  if (hasColumns(row, columns)) return row;
+  throw new TypeError(
+    `Unexpected values in columns ${mismatchedColumns(row, columns).join(", ")} of a row from: ${oneLine(sql2)}`
+  );
+}
+function oneLine(sql2) {
+  return sql2.replace(/\s+/g, " ").trim();
+}
+function isRecord4(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isRunSummary(value) {
+  return isRecord4(value) && typeof value.totalCases === "number" && typeof value.passCount === "number" && typeof value.failCount === "number" && typeof value.totalLatencyMs === "number" && typeof value.totalTokensIn === "number" && typeof value.totalTokensOut === "number" && isRecord4(value.meanScores) && Object.values(value.meanScores).every((score) => typeof score === "number");
+}
+function parseJson2(text, guard, source) {
+  const value = JSON.parse(text);
+  if (guard(value)) return value;
+  throw new TypeError(`Unexpected JSON in ${source}`);
+}
+var suiteColumns = {
+  id: "text",
+  name: "text",
+  created_at: "number"
+};
+var runColumns = {
+  id: "text",
+  suite_id: "text",
+  name: "text",
+  model: "text",
+  config: "text?",
+  started_at: "number",
+  finished_at: "number?",
+  status: "text",
+  summary: "text?"
+};
+var caseColumns = {
+  id: "text",
+  run_id: "text",
+  idx: "number",
+  input: "text",
+  output: "text?",
+  expected: "text?",
+  latency_ms: "number?",
+  tokens_in: "number?",
+  tokens_out: "number?",
+  error: "text?"
+};
+var promptColumns = {
+  id: "text",
+  name: "text",
+  version: "number",
+  content: "text",
+  created_at: "number"
+};
+function toSuite(row) {
+  return { id: row.id, name: row.name, created_at: row.created_at };
+}
+function toRun(row) {
+  if (!isRunStatus(row.status)) {
+    throw new TypeError(`Unexpected status "${row.status}" of run ${row.id}`);
+  }
+  return {
+    id: row.id,
+    suite_id: row.suite_id,
+    name: row.name,
+    model: row.model,
+    config: row.config ? parseJson2(row.config, isRecord4, `config of run ${row.id}`) : null,
+    started_at: row.started_at,
+    finished_at: row.finished_at,
+    status: row.status,
+    summary: row.summary ? parseJson2(row.summary, isRunSummary, `summary of run ${row.id}`) : null
+  };
+}
+function toCase(row) {
+  const input = JSON.parse(row.input);
+  const expected = row.expected ? JSON.parse(row.expected) : null;
+  return {
+    id: row.id,
+    run_id: row.run_id,
+    idx: row.idx,
+    input,
+    output: row.output,
+    expected,
+    latency_ms: row.latency_ms,
+    tokens_in: row.tokens_in,
+    tokens_out: row.tokens_out,
+    error: row.error
+  };
+}
+function toPrompt(row) {
+  return {
+    id: row.id,
+    name: row.name,
+    version: row.version,
+    content: row.content,
+    created_at: row.created_at
+  };
+}
 var RunStore = class {
   #db;
   #statements = /* @__PURE__ */ new Map();
@@ -626,6 +787,21 @@ var RunStore = class {
     }
     return stmt;
   }
+  #get(columns, sql2, ...params) {
+    const row = this.#stmt(sql2).get(...params);
+    return row === void 0 ? void 0 : readRow(row, columns, sql2);
+  }
+  /** Reads an aggregate query, which returns exactly one row. */
+  #one(columns, sql2, ...params) {
+    const row = this.#get(columns, sql2, ...params);
+    if (row === void 0) {
+      throw new Error(`Expected one row from: ${oneLine(sql2)}`);
+    }
+    return row;
+  }
+  #all(columns, sql2, ...params) {
+    return this.#stmt(sql2).all(...params).map((row) => readRow(row, columns, sql2));
+  }
   #transaction(fn) {
     this.#db.exec("BEGIN TRANSACTION");
     try {
@@ -633,7 +809,10 @@ var RunStore = class {
       this.#db.exec("COMMIT");
       return result;
     } catch (error) {
-      this.#db.exec("ROLLBACK");
+      try {
+        this.#db.exec("ROLLBACK");
+      } catch {
+      }
       throw error;
     }
   }
@@ -653,7 +832,7 @@ var RunStore = class {
     );
   }
   #migratePromptsTableIfNeeded() {
-    const columns = this.#stmt("PRAGMA table_info(prompts)").all();
+    const columns = this.#all({ name: "text" }, "PRAGMA table_info(prompts)");
     if (columns.length === 0) return;
     if (columns.some((column) => column.name === "version")) return;
     this.#transaction(() => {
@@ -683,13 +862,17 @@ var RunStore = class {
     });
   }
   #migrateRunsTableToSuiteRequired() {
-    const runColumns = this.#stmt("PRAGMA table_info(runs)").all();
-    if (runColumns.length === 0) return;
-    const suiteColumn = runColumns.find((column) => column.name === "suite_id");
+    const columns = this.#all(
+      { name: "text", notnull: "number" },
+      "PRAGMA table_info(runs)"
+    );
+    if (columns.length === 0) return;
+    const suiteColumn = columns.find((column) => column.name === "suite_id");
     const hasNonNullSuite = suiteColumn?.notnull === 1;
-    const runForeignKeys = this.#stmt(
+    const runForeignKeys = this.#all(
+      { from: "text", on_delete: "text", table: "text" },
       "PRAGMA foreign_key_list(runs)"
-    ).all();
+    );
     const suiteForeignKey = runForeignKeys.find(
       (fk) => fk.from === "suite_id" && fk.table === "suites"
     );
@@ -738,8 +921,12 @@ var RunStore = class {
     return { id, name, created_at: now };
   }
   getSuite(id) {
-    const row = this.#stmt("SELECT * FROM suites WHERE id = ?").get(id);
-    return row ?? void 0;
+    const row = this.#get(
+      suiteColumns,
+      "SELECT * FROM suites WHERE id = ?",
+      id
+    );
+    return row ? toSuite(row) : void 0;
   }
   renameSuite(id, name) {
     this.#stmt("UPDATE suites SET name = ? WHERE id = ?").run(name, id);
@@ -799,77 +986,45 @@ var RunStore = class {
     });
   }
   getRun(runId) {
-    const row = this.#stmt("SELECT * FROM runs WHERE id = ?").get(runId);
-    if (!row) return void 0;
-    return {
-      id: row.id,
-      suite_id: row.suite_id,
-      name: row.name,
-      model: row.model,
-      config: row.config ? JSON.parse(row.config) : null,
-      started_at: row.started_at,
-      finished_at: row.finished_at,
-      status: row.status,
-      summary: row.summary ? JSON.parse(row.summary) : null
-    };
+    const row = this.#get(runColumns, "SELECT * FROM runs WHERE id = ?", runId);
+    return row ? toRun(row) : void 0;
   }
   listRuns(suiteId) {
-    const sql2 = suiteId ? "SELECT * FROM runs WHERE suite_id = ? ORDER BY started_at" : "SELECT * FROM runs ORDER BY started_at";
-    const rows = suiteId ? this.#stmt(sql2).all(suiteId) : this.#stmt(sql2).all();
-    return rows.map((row) => ({
-      id: row.id,
-      suite_id: row.suite_id,
-      name: row.name,
-      model: row.model,
-      config: row.config ? JSON.parse(row.config) : null,
-      started_at: row.started_at,
-      finished_at: row.finished_at,
-      status: row.status,
-      summary: row.summary ? JSON.parse(row.summary) : null
-    }));
+    const rows = suiteId ? this.#all(
+      runColumns,
+      "SELECT * FROM runs WHERE suite_id = ? ORDER BY started_at",
+      suiteId
+    ) : this.#all(runColumns, "SELECT * FROM runs ORDER BY started_at");
+    return rows.map(toRun);
   }
   getCases(runId) {
-    const rows = this.#stmt(
-      "SELECT * FROM cases WHERE run_id = ? ORDER BY idx"
-    ).all(runId);
-    return rows.map((row) => ({
-      id: row.id,
-      run_id: row.run_id,
-      idx: row.idx,
-      input: JSON.parse(row.input),
-      output: row.output,
-      expected: row.expected ? JSON.parse(row.expected) : null,
-      latency_ms: row.latency_ms,
-      tokens_in: row.tokens_in,
-      tokens_out: row.tokens_out,
-      error: row.error
-    }));
+    return this.#all(
+      caseColumns,
+      "SELECT * FROM cases WHERE run_id = ? ORDER BY idx",
+      runId
+    ).map(toCase);
   }
   getFailingCases(runId, threshold = 0.5) {
-    const rows = this.#stmt(
+    const rows = this.#all(
+      {
+        ...caseColumns,
+        scorer_name: "text",
+        score: "number",
+        score_reason: "text?"
+      },
       `SELECT c.*, s.scorer_name, s.score, s.reason as score_reason
        FROM cases c
        JOIN scores s ON s.case_id = c.id
        WHERE c.run_id = ? AND s.score < ?
-       ORDER BY c.idx`
-    ).all(runId, threshold);
+       ORDER BY c.idx`,
+      runId,
+      threshold
+    );
     const caseMap = /* @__PURE__ */ new Map();
     for (const row of rows) {
       let c = caseMap.get(row.id);
       if (!c) {
-        c = {
-          id: row.id,
-          run_id: row.run_id,
-          idx: row.idx,
-          input: JSON.parse(row.input),
-          output: row.output,
-          expected: row.expected ? JSON.parse(row.expected) : null,
-          latency_ms: row.latency_ms,
-          tokens_in: row.tokens_in,
-          tokens_out: row.tokens_out,
-          error: row.error,
-          scores: []
-        };
+        c = { ...toCase(row), scores: [] };
         caseMap.set(row.id, c);
       }
       c.scores.push({
@@ -881,33 +1036,44 @@ var RunStore = class {
     return Array.from(caseMap.values());
   }
   getRunSummary(runId, threshold = 0.5) {
-    const totals = this.#stmt(
+    const totals = this.#one(
+      {
+        totalCases: "number",
+        totalLatencyMs: "number",
+        totalTokensIn: "number",
+        totalTokensOut: "number"
+      },
       `SELECT
         COUNT(DISTINCT c.id) as totalCases,
         COALESCE(SUM(c.latency_ms), 0) as totalLatencyMs,
         COALESCE(SUM(c.tokens_in), 0) as totalTokensIn,
         COALESCE(SUM(c.tokens_out), 0) as totalTokensOut
-       FROM cases c WHERE c.run_id = ?`
-    ).get(runId);
-    const scorerMeans = this.#stmt(
+       FROM cases c WHERE c.run_id = ?`,
+      runId
+    );
+    const scorerMeans = this.#all(
+      { scorer_name: "text", meanScore: "number" },
       `SELECT s.scorer_name, AVG(s.score) as meanScore
        FROM scores s
        JOIN cases c ON c.id = s.case_id
        WHERE c.run_id = ?
-       GROUP BY s.scorer_name`
-    ).all(runId);
+       GROUP BY s.scorer_name`,
+      runId
+    );
     const meanScores = {};
     for (const row of scorerMeans) {
       meanScores[row.scorer_name] = row.meanScore;
     }
-    const passFail = this.#stmt(
+    const passFail = this.#all(
+      { id: "text", minScore: "number" },
       `SELECT c.id,
         MIN(s.score) as minScore
        FROM cases c
        JOIN scores s ON s.case_id = c.id
        WHERE c.run_id = ?
-       GROUP BY c.id`
-    ).all(runId);
+       GROUP BY c.id`,
+      runId
+    );
     let passCount = 0;
     let failCount = 0;
     for (const row of passFail) {
@@ -925,71 +1091,61 @@ var RunStore = class {
     };
   }
   findSuiteByName(name) {
-    const row = this.#stmt(
-      "SELECT * FROM suites WHERE name = ? ORDER BY created_at DESC LIMIT 1"
-    ).get(name);
-    return row ?? void 0;
+    const row = this.#get(
+      suiteColumns,
+      "SELECT * FROM suites WHERE name = ? ORDER BY created_at DESC LIMIT 1",
+      name
+    );
+    return row ? toSuite(row) : void 0;
   }
   getLatestCompletedRun(suiteId, model) {
-    const sql2 = model ? "SELECT * FROM runs WHERE suite_id = ? AND status = ? AND model = ? ORDER BY started_at DESC LIMIT 1" : "SELECT * FROM runs WHERE suite_id = ? AND status = ? ORDER BY started_at DESC LIMIT 1";
-    const row = model ? this.#stmt(sql2).get(suiteId, "completed", model) : this.#stmt(sql2).get(suiteId, "completed");
-    if (!row) return void 0;
-    return {
-      id: row.id,
-      suite_id: row.suite_id,
-      name: row.name,
-      model: row.model,
-      config: row.config ? JSON.parse(row.config) : null,
-      started_at: row.started_at,
-      finished_at: row.finished_at,
-      status: row.status,
-      summary: row.summary ? JSON.parse(row.summary) : null
-    };
+    const row = model ? this.#get(
+      runColumns,
+      "SELECT * FROM runs WHERE suite_id = ? AND status = ? AND model = ? ORDER BY started_at DESC LIMIT 1",
+      suiteId,
+      "completed",
+      model
+    ) : this.#get(
+      runColumns,
+      "SELECT * FROM runs WHERE suite_id = ? AND status = ? ORDER BY started_at DESC LIMIT 1",
+      suiteId,
+      "completed"
+    );
+    return row ? toRun(row) : void 0;
   }
   listSuites() {
-    const rows = this.#stmt(
+    return this.#all(
+      suiteColumns,
       "SELECT * FROM suites ORDER BY created_at DESC"
-    ).all();
-    return rows.map((row) => ({
-      id: row.id,
-      name: row.name,
-      created_at: row.created_at
-    }));
+    ).map(toSuite);
   }
   createPrompt(name, content) {
     const id = crypto.randomUUID();
     const now = Date.now();
-    const latest = this.#stmt(
-      "SELECT MAX(version) as latestVersion FROM prompts WHERE name = ?"
-    ).get(name);
-    const version = (latest?.latestVersion ?? 0) + 1;
+    const { latestVersion } = this.#one(
+      { latestVersion: "number?" },
+      "SELECT MAX(version) as latestVersion FROM prompts WHERE name = ?",
+      name
+    );
+    const version = (latestVersion ?? 0) + 1;
     this.#stmt(
       "INSERT INTO prompts (id, name, version, content, created_at) VALUES (?, ?, ?, ?, ?)"
     ).run(id, name, version, content, now);
     return { id, name, version, content, created_at: now };
   }
   listPrompts() {
-    const rows = this.#stmt(
+    return this.#all(
+      promptColumns,
       "SELECT * FROM prompts ORDER BY name COLLATE NOCASE ASC, version DESC"
-    ).all();
-    return rows.map((row) => ({
-      id: row.id,
-      name: row.name,
-      version: row.version,
-      content: row.content,
-      created_at: row.created_at
-    }));
+    ).map(toPrompt);
   }
   getPrompt(id) {
-    const row = this.#stmt("SELECT * FROM prompts WHERE id = ?").get(id);
-    if (!row) return void 0;
-    return {
-      id: row.id,
-      name: row.name,
-      version: row.version,
-      content: row.content,
-      created_at: row.created_at
-    };
+    const row = this.#get(
+      promptColumns,
+      "SELECT * FROM prompts WHERE id = ?",
+      id
+    );
+    return row ? toPrompt(row) : void 0;
   }
   deletePrompt(id) {
     this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
@@ -1114,6 +1270,9 @@ async function wrapTask(task, input, timeoutMs) {
     };
   }
 }
+function expectedOf(input) {
+  return typeof input === "object" && input !== null && "expected" in input ? input.expected : void 0;
+}
 function clampScore(score, scorerName) {
   if (score < 0 || score > 1) {
     console.warn(
@@ -1161,6 +1320,7 @@ async function runEval(config) {
     await semaphore.acquire();
     try {
       emitter.emit("case:start", { runId, index, input });
+      const expected = expectedOf(input);
       let finalResult;
       let finalScores;
       if (trials > 1) {
@@ -1178,7 +1338,7 @@ async function runEval(config) {
               const sr = await scorer({
                 input,
                 output: result.output,
-                expected: input.expected
+                expected
               });
               scores[sName] = {
                 score: clampScore(sr.score, sName),
@@ -1223,7 +1383,7 @@ async function runEval(config) {
             const sr = await scorer({
               input,
               output: finalResult.output,
-              expected: input.expected
+              expected
             });
             finalScores[sName] = {
               score: clampScore(sr.score, sName),
@@ -1240,7 +1400,7 @@ async function runEval(config) {
         idx: index,
         input,
         output: finalResult.output || null,
-        expected: input.expected,
+        expected,
         latency_ms: finalResult.latencyMs,
         tokens_in: finalResult.tokensIn,
         tokens_out: finalResult.tokensOut,
@@ -1276,7 +1436,7 @@ async function runEval(config) {
         index,
         input,
         output: finalResult.output,
-        expected: input.expected,
+        expected,
         scores: finalScores,
         error: finalResult.error,
         latencyMs: finalResult.latencyMs,
@@ -1721,21 +1881,31 @@ function renderFailuresByScorer(cases, threshold) {
 }
 
 // packages/evals/src/reporters/json.ts
-import { appendFile, mkdir as mkdir2 } from "node:fs/promises";
+import { appendFileSync, mkdirSync as mkdirSync2 } from "node:fs";
 function jsonReporter(options) {
   const outputDir = resolveOutputDir(options?.outputDir);
   const pretty = options?.pretty ?? true;
-  let streamFilename = "";
+  const streamFiles = /* @__PURE__ */ new Map();
   return {
-    async onRunStart(data) {
-      await mkdir2(outputDir, { recursive: true });
-      streamFilename = getReportPath(outputDir, data.name, data.runId, "jsonl");
+    onRunStart(data) {
+      mkdirSync2(outputDir, { recursive: true });
+      streamFiles.set(
+        data.runId,
+        getReportPath(outputDir, data.name, data.runId, "jsonl")
+      );
     },
-    async onCaseEnd(data) {
+    onCaseEnd(data) {
+      const streamFile = streamFiles.get(data.runId);
+      if (streamFile === void 0) {
+        throw new Error(
+          `jsonReporter received a case for run ${data.runId} before its onRunStart`
+        );
+      }
       const line = stringifyUnknown(data, { space: 0, fallback: "null" });
-      await appendFile(streamFilename, line + "\n", "utf-8");
+      appendFileSync(streamFile, line + "\n", "utf-8");
     },
     async onRunEnd(data) {
+      streamFiles.delete(data.runId);
       const content = stringifyUnknown(data, {
         space: pretty ? 2 : 0,
         fallback: "null"
@@ -2015,12 +2185,11 @@ function resolveFailedIndexes(store, suiteName, model, threshold) {
   return new Set(failingCases.map((c) => c.idx));
 }
 var EvalBuilder = class {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  #options;
+  #plan;
   #selection = { type: "all" };
   #shouldAssert = false;
-  constructor(options) {
-    this.#options = options;
+  constructor(plan) {
+    this.#plan = plan;
   }
   #setSelection(selection) {
     if (this.#selection.type !== "all") {
@@ -2052,77 +2221,76 @@ var EvalBuilder = class {
     return this.#execute().then(onfulfilled, onrejected);
   }
   async #execute() {
-    if ("models" in this.#options) {
-      return this.#executeMulti();
-    }
-    return this.#executeSingle();
-  }
-  #applyDatasetFilter(ds) {
-    switch (this.#selection.type) {
-      case "all":
-        return ds;
-      case "cases":
-        return this.#selection.indexes.size > 0 ? filterRecordsByIndex(ds, this.#selection.indexes) : ds;
-      case "sample":
-        return dataset(ds).sample(this.#selection.count);
-      case "failed":
-        return ds;
-    }
-  }
-  async #executeSingle() {
-    const options = this.#options;
-    let ds = options.dataset;
-    if (this.#selection.type === "failed") {
-      const indexes = resolveFailedIndexes(
-        options.store,
-        options.name,
-        options.model,
-        options.threshold
-      );
-      if (indexes.size > 0) {
-        ds = filterRecordsByIndex(ds, indexes);
-      }
-    } else {
-      ds = this.#applyDatasetFilter(ds);
-    }
-    const result = await evaluateSingle({ ...options, dataset: ds });
-    if (this.#shouldAssert && result.failCount > 0) {
+    const result = await this.#plan.run(this.#selection);
+    if (this.#shouldAssert && this.#plan.hasFailures(result)) {
       throw new EvalAssertionError(result);
     }
     return result;
   }
-  async #executeMulti() {
-    const options = this.#options;
-    let result;
-    if (this.#selection.type === "failed") {
-      const perModelIndexes = /* @__PURE__ */ new Map();
-      for (const variant of options.models) {
-        perModelIndexes.set(
-          variant.name,
-          resolveFailedIndexes(
-            options.store,
-            options.name,
-            variant.name,
-            options.threshold
-          )
-        );
-      }
-      result = await evaluateEach(options, perModelIndexes);
-    } else {
-      const filtered = this.#applyDatasetFilter(options.dataset);
-      result = await evaluateEach({ ...options, dataset: filtered });
-    }
-    if (this.#shouldAssert && result.some((s) => s.failCount > 0)) {
-      throw new EvalAssertionError(result);
+};
+function selectCases(ds, selection) {
+  switch (selection.type) {
+    case "all":
+      return ds;
+    case "cases":
+      return selection.indexes.size > 0 ? filterRecordsByIndex(ds, selection.indexes) : ds;
+    case "sample":
+      return dataset(ds).sample(selection.count);
+  }
+}
+var SingleModelEval = class {
+  #options;
+  constructor(options) {
+    this.#options = options;
+  }
+  run(selection) {
+    return evaluateSingle({
+      ...this.#options,
+      dataset: this.#cases(selection)
+    });
+  }
+  hasFailures(summary) {
+    return summary.failCount > 0;
+  }
+  #cases(selection) {
+    const { dataset: ds, store, name, model, threshold } = this.#options;
+    if (selection.type !== "failed") return selectCases(ds, selection);
+    const indexes = resolveFailedIndexes(store, name, model, threshold);
+    return indexes.size > 0 ? filterRecordsByIndex(ds, indexes) : ds;
+  }
+};
+var EachModelEval = class {
+  #options;
+  constructor(options) {
+    this.#options = options;
+  }
+  run(selection) {
+    if (selection.type === "failed") {
+      return evaluateEach(this.#options, this.#failedIndexesPerModel());
     }
-    return result;
+    return evaluateEach({
+      ...this.#options,
+      dataset: selectCases(this.#options.dataset, selection)
+    });
+  }
+  hasFailures(summaries) {
+    return summaries.some((s) => s.failCount > 0);
+  }
+  #failedIndexesPerModel() {
+    const { models, store, name, threshold } = this.#options;
+    return new Map(
+      models.map((variant) => [
+        variant.name,
+        resolveFailedIndexes(store, name, variant.name, threshold)
+      ])
+    );
   }
 };
 function evaluate(options) {
   if ("models" in options) {
-    return new EvalBuilder(options);
+    return new EvalBuilder(new EachModelEval(options));
   }
-  return new EvalBuilder(options);
+  return new EvalBuilder(new SingleModelEval(options));
 }
 function wireReporters(reporters) {
   const emitter = new EvalEmitter();