@deepagents/evals 0.19.0 → 0.22.0

package/dist/evaluate/index.js

@@ -3,6 +3,68 @@ import { createReadStream } from "node:fs";
 import { readFile } from "node:fs/promises";
 import { extname } from "node:path";
 import { createInterface } from "node:readline";
+
+// packages/evals/src/dataset/record-selection.ts
+function parsePositiveInt(token) {
+  if (!/^\d+$/.test(token)) {
+    throw new Error(`Invalid record token "${token}"`);
+  }
+  const value = Number(token);
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`Record numbers must be >= 1. Received "${token}"`);
+  }
+  return value;
+}
+function parseRecordSelection(spec) {
+  const trimmed = spec.trim();
+  if (!trimmed) {
+    return { indexes: /* @__PURE__ */ new Set(), normalized: "" };
+  }
+  const indexes = /* @__PURE__ */ new Set();
+  const parts = trimmed.split(",").map((part) => part.trim()).filter(Boolean);
+  if (parts.length === 0) {
+    throw new Error("Record selection is empty.");
+  }
+  for (const part of parts) {
+    const rangeMatch = /^(\d+)\s*-\s*(\d+)$/.exec(part);
+    if (rangeMatch) {
+      const start = parsePositiveInt(rangeMatch[1]);
+      const end = parsePositiveInt(rangeMatch[2]);
+      if (end < start) {
+        throw new Error(
+          `Invalid range "${part}". Range end must be >= range start.`
+        );
+      }
+      for (let i = start; i <= end; i++) {
+        indexes.add(i - 1);
+      }
+      continue;
+    }
+    const value = parsePositiveInt(part);
+    indexes.add(value - 1);
+  }
+  return {
+    indexes,
+    normalized: Array.from(indexes).sort((a, b) => a - b).map((i) => String(i + 1)).join(",")
+  };
+}
+async function* filterRecordsByIndex(source, indexes) {
+  if (indexes.size === 0) {
+    for await (const item of source) {
+      yield item;
+    }
+    return;
+  }
+  let idx = 0;
+  for await (const item of source) {
+    if (indexes.has(idx)) {
+      yield item;
+    }
+    idx++;
+  }
+}
+
+// packages/evals/src/dataset/index.ts
 var Dataset = class _Dataset {
   #source;
   constructor(source) {
@@ -70,6 +132,22 @@ var Dataset = class _Dataset {
       }
     });
   }
+  pick(indexes) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      if (indexes.size === 0) {
+        yield* source();
+        return;
+      }
+      let idx = 0;
+      for await (const item of source()) {
+        if (indexes.has(idx)) {
+          yield item;
+        }
+        idx++;
+      }
+    });
+  }
   async toArray() {
     const result = [];
     for await (const item of this.#source()) {
@@ -361,7 +439,8 @@ async function runEval(config) {
               });
               scores[sName] = {
                 score: clampScore(sr.score, sName),
-                reason: sr.reason
+                reason: sr.reason,
+                metadata: sr.metadata
               };
             }
             trialResults.push({ result, scores });
@@ -387,7 +466,8 @@ async function runEval(config) {
           const meanScore = trialResults.reduce((sum, t) => sum + t.scores[sName].score, 0) / trials;
           finalScores[sName] = {
             score: meanScore,
-            reason: trialResults[trialResults.length - 1].scores[sName]?.reason
+            reason: trialResults[trialResults.length - 1].scores[sName]?.reason,
+            metadata: trialResults[trialResults.length - 1].scores[sName]?.metadata
           };
         }
       } else {
@@ -404,7 +484,8 @@ async function runEval(config) {
             });
             finalScores[sName] = {
               score: clampScore(sr.score, sName),
-              reason: sr.reason
+              reason: sr.reason,
+              metadata: sr.metadata
             };
           }
         }
@@ -520,372 +601,152 @@ function computeSummary(cases, scorerNames, threshold) {
 }
 
 // packages/evals/src/store/index.ts
-import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
 import { DatabaseSync } from "node:sqlite";
 
-// packages/evals/src/store/ddl.sqlite.sql
-var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
-
-// packages/evals/src/store/index.ts
-var RunStore = class {
-  #db;
-  #statements = /* @__PURE__ */ new Map();
-  #stmt(sql) {
-    let stmt = this.#statements.get(sql);
-    if (!stmt) {
-      stmt = this.#db.prepare(sql);
-      this.#statements.set(sql, stmt);
-    }
-    return stmt;
-  }
-  #transaction(fn) {
-    this.#db.exec("BEGIN TRANSACTION");
-    try {
-      const result = fn();
-      this.#db.exec("COMMIT");
-      return result;
-    } catch (error) {
-      this.#db.exec("ROLLBACK");
-      throw error;
-    }
+// packages/evals/src/evaluate/index.ts
+var EvalAssertionError = class extends Error {
+  summary;
+  constructor(summary) {
+    const msg = Array.isArray(summary) ? `Eval assertion failed: ${summary.filter((s) => s.failCount > 0).length} of ${summary.length} model runs have failures` : `Eval assertion failed: ${summary.failCount} of ${summary.totalCases} cases failed`;
+    super(msg);
+    this.name = "EvalAssertionError";
+    this.summary = summary;
   }
-  constructor(pathOrDb) {
-    if (pathOrDb instanceof DatabaseSync) {
-      this.#db = pathOrDb;
-    } else {
-      const dbPath = pathOrDb ?? ".evals/store.db";
-      mkdirSync(dirname(dbPath), { recursive: true });
-      this.#db = new DatabaseSync(dbPath);
-    }
-    this.#db.exec(ddl_sqlite_default);
-    this.#migrateRunsTableToSuiteRequired();
-    this.#migratePromptsTableIfNeeded();
-    this.#db.exec(
-      "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+};
+function resolveFailedIndexes(store, suiteName, model, threshold) {
+  const suite = store.findSuiteByName(suiteName);
+  if (!suite) {
+    console.warn(
+      `No previous suite found for '${suiteName}'. Running all cases.`
     );
+    return /* @__PURE__ */ new Set();
   }
-  #migratePromptsTableIfNeeded() {
-    const columns = this.#stmt("PRAGMA table_info(prompts)").all();
-    if (columns.length === 0) return;
-    if (columns.some((column) => column.name === "version")) return;
-    this.#transaction(() => {
-      this.#db.exec("ALTER TABLE prompts RENAME TO prompts_legacy");
-      this.#db.exec(`
-        CREATE TABLE prompts (
-          id TEXT PRIMARY KEY,
-          name TEXT NOT NULL,
-          version INTEGER NOT NULL,
-          content TEXT NOT NULL,
-          created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
-          UNIQUE(name, version)
-        )
-      `);
-      this.#db.exec(`
-        INSERT INTO prompts (id, name, version, content, created_at)
-        SELECT id, name, 1, content, created_at
-        FROM prompts_legacy
-      `);
-      this.#db.exec("DROP TABLE prompts_legacy");
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at)"
-      );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
-      );
-    });
-  }
-  #migrateRunsTableToSuiteRequired() {
-    const runColumns = this.#stmt("PRAGMA table_info(runs)").all();
-    if (runColumns.length === 0) return;
-    const suiteColumn = runColumns.find((column) => column.name === "suite_id");
-    const hasNonNullSuite = suiteColumn?.notnull === 1;
-    const runForeignKeys = this.#stmt(
-      "PRAGMA foreign_key_list(runs)"
-    ).all();
-    const suiteForeignKey = runForeignKeys.find(
-      (fk) => fk.from === "suite_id" && fk.table === "suites"
-    );
-    const hasCascadeDelete = suiteForeignKey?.on_delete === "CASCADE";
-    if (hasNonNullSuite && hasCascadeDelete) return;
-    this.#statements.clear();
-    this.#transaction(() => {
-      this.#db.exec(`
-        CREATE TABLE runs_next (
-          id TEXT PRIMARY KEY,
-          suite_id TEXT NOT NULL,
-          name TEXT NOT NULL,
-          model TEXT NOT NULL,
-          config TEXT,
-          started_at INTEGER NOT NULL,
-          finished_at INTEGER,
-          status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),
-          summary TEXT,
-          FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE
-        )
-      `);
-      this.#db.exec("DELETE FROM runs WHERE suite_id IS NULL");
-      this.#db.exec(`
-        INSERT INTO runs_next (id, suite_id, name, model, config, started_at, finished_at, status, summary)
-        SELECT r.id, r.suite_id, r.name, r.model, r.config, r.started_at, r.finished_at, r.status, r.summary
-        FROM runs r
-        JOIN suites s ON s.id = r.suite_id
-      `);
-      this.#db.exec("DROP TABLE runs");
-      this.#db.exec("ALTER TABLE runs_next RENAME TO runs");
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id)"
-      );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at)"
-      );
-    });
-    this.#statements.clear();
-  }
-  createSuite(name) {
-    const id = crypto.randomUUID();
-    const now = Date.now();
-    this.#stmt(
-      "INSERT INTO suites (id, name, created_at) VALUES (?, ?, ?)"
-    ).run(id, name, now);
-    return { id, name, created_at: now };
-  }
-  createRun(run) {
-    const id = crypto.randomUUID();
-    const now = Date.now();
-    this.#stmt(
-      "INSERT INTO runs (id, suite_id, name, model, config, started_at) VALUES (?, ?, ?, ?, ?, ?)"
-    ).run(
-      id,
-      run.suite_id,
-      run.name,
-      run.model,
-      run.config ? JSON.stringify(run.config) : null,
-      now
+  const run = store.getLatestCompletedRun(suite.id, model);
+  if (!run) {
+    console.warn(
+      `No previous completed run found for '${suiteName}'${model ? ` [${model}]` : ""}. Running all cases.`
     );
-    return id;
+    return /* @__PURE__ */ new Set();
   }
-  finishRun(runId, status, summary) {
-    this.#stmt(
-      "UPDATE runs SET finished_at = ?, status = ?, summary = ? WHERE id = ?"
-    ).run(Date.now(), status, summary ? JSON.stringify(summary) : null, runId);
+  const failingCases = store.getFailingCases(run.id, threshold);
+  if (failingCases.length === 0) {
+    console.warn(`No failed cases in previous run. Running all cases.`);
+    return /* @__PURE__ */ new Set();
   }
-  saveCases(cases) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+  console.warn(
+    `Retrying ${failingCases.length} failed cases from previous run`
+  );
+  return new Set(failingCases.map((c) => c.idx));
+}
+var EvalBuilder = class {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  #options;
+  #selection = { type: "all" };
+  #shouldAssert = false;
+  constructor(options) {
+    this.#options = options;
+  }
+  #setSelection(selection) {
+    if (this.#selection.type !== "all") {
+      throw new Error(
+        `Cannot combine .${this.#selection.type}() with .${selection.type}()`
       );
-      for (const c of cases) {
-        stmt.run(
-          c.id,
-          c.run_id,
-          c.idx,
-          JSON.stringify(c.input),
-          c.output,
-          c.expected != null ? JSON.stringify(c.expected) : null,
-          c.latency_ms,
-          c.tokens_in,
-          c.tokens_out,
-          c.error ?? null
-        );
-      }
-    });
+    }
+    this.#selection = selection;
+    return this;
   }
-  saveScores(scores) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
-      );
-      for (const s of scores) {
-        stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
-      }
-    });
+  failed() {
+    return this.#setSelection({ type: "failed" });
   }
-  getRun(runId) {
-    const row = this.#stmt("SELECT * FROM runs WHERE id = ?").get(runId);
-    if (!row) return void 0;
-    return {
-      id: row.id,
-      suite_id: row.suite_id,
-      name: row.name,
-      model: row.model,
-      config: row.config ? JSON.parse(row.config) : null,
-      started_at: row.started_at,
-      finished_at: row.finished_at,
-      status: row.status,
-      summary: row.summary ? JSON.parse(row.summary) : null
-    };
+  cases(spec) {
+    const { indexes } = parseRecordSelection(spec);
+    return this.#setSelection({ type: "cases", indexes });
   }
-  listRuns(suiteId) {
-    const sql = suiteId ? "SELECT * FROM runs WHERE suite_id = ? ORDER BY started_at" : "SELECT * FROM runs ORDER BY started_at";
-    const rows = suiteId ? this.#stmt(sql).all(suiteId) : this.#stmt(sql).all();
-    return rows.map((row) => ({
-      id: row.id,
-      suite_id: row.suite_id,
-      name: row.name,
-      model: row.model,
-      config: row.config ? JSON.parse(row.config) : null,
-      started_at: row.started_at,
-      finished_at: row.finished_at,
-      status: row.status,
-      summary: row.summary ? JSON.parse(row.summary) : null
-    }));
-  }
-  getCases(runId) {
-    const rows = this.#stmt(
-      "SELECT * FROM cases WHERE run_id = ? ORDER BY idx"
-    ).all(runId);
-    return rows.map((row) => ({
-      id: row.id,
-      run_id: row.run_id,
-      idx: row.idx,
-      input: JSON.parse(row.input),
-      output: row.output,
-      expected: row.expected ? JSON.parse(row.expected) : null,
-      latency_ms: row.latency_ms,
-      tokens_in: row.tokens_in,
-      tokens_out: row.tokens_out,
-      error: row.error
-    }));
-  }
-  getFailingCases(runId, threshold = 0.5) {
-    const rows = this.#stmt(
-      `SELECT c.*, s.scorer_name, s.score, s.reason as score_reason
-       FROM cases c
-       JOIN scores s ON s.case_id = c.id
-       WHERE c.run_id = ? AND s.score < ?
-       ORDER BY c.idx`
-    ).all(runId, threshold);
-    const caseMap = /* @__PURE__ */ new Map();
-    for (const row of rows) {
-      let c = caseMap.get(row.id);
-      if (!c) {
-        c = {
-          id: row.id,
-          run_id: row.run_id,
-          idx: row.idx,
-          input: JSON.parse(row.input),
-          output: row.output,
-          expected: row.expected ? JSON.parse(row.expected) : null,
-          latency_ms: row.latency_ms,
-          tokens_in: row.tokens_in,
-          tokens_out: row.tokens_out,
-          error: row.error,
-          scores: []
-        };
-        caseMap.set(row.id, c);
-      }
-      c.scores.push({
-        scorer_name: row.scorer_name,
-        score: row.score,
-        reason: row.score_reason
-      });
+  sample(count) {
+    if (count < 1) {
+      throw new Error("Sample count must be >= 1");
     }
-    return Array.from(caseMap.values());
-  }
-  getRunSummary(runId, threshold = 0.5) {
-    const totals = this.#stmt(
-      `SELECT
-        COUNT(DISTINCT c.id) as totalCases,
-        COALESCE(SUM(c.latency_ms), 0) as totalLatencyMs,
-        COALESCE(SUM(c.tokens_in), 0) as totalTokensIn,
-        COALESCE(SUM(c.tokens_out), 0) as totalTokensOut
-       FROM cases c WHERE c.run_id = ?`
-    ).get(runId);
-    const scorerMeans = this.#stmt(
-      `SELECT s.scorer_name, AVG(s.score) as meanScore
-       FROM scores s
-       JOIN cases c ON c.id = s.case_id
-       WHERE c.run_id = ?
-       GROUP BY s.scorer_name`
-    ).all(runId);
-    const meanScores = {};
-    for (const row of scorerMeans) {
-      meanScores[row.scorer_name] = row.meanScore;
+    return this.#setSelection({ type: "sample", count });
+  }
+  assert() {
+    this.#shouldAssert = true;
+    return this;
+  }
+  then(onfulfilled, onrejected) {
+    return this.#execute().then(onfulfilled, onrejected);
+  }
+  async #execute() {
+    if ("models" in this.#options) {
+      return this.#executeMulti();
     }
-    const passFail = this.#stmt(
-      `SELECT c.id,
-        MIN(s.score) as minScore
-       FROM cases c
-       JOIN scores s ON s.case_id = c.id
-       WHERE c.run_id = ?
-       GROUP BY c.id`
-    ).all(runId);
-    let passCount = 0;
-    let failCount = 0;
-    for (const row of passFail) {
-      if (row.minScore >= threshold) passCount++;
-      else failCount++;
+    return this.#executeSingle();
+  }
+  #applyDatasetFilter(ds) {
+    switch (this.#selection.type) {
+      case "all":
+        return ds;
+      case "cases":
+        return this.#selection.indexes.size > 0 ? filterRecordsByIndex(ds, this.#selection.indexes) : ds;
+      case "sample":
+        return dataset(ds).sample(this.#selection.count);
+      case "failed":
+        return ds;
     }
-    return {
-      totalCases: totals.totalCases,
-      passCount,
-      failCount,
-      meanScores,
-      totalLatencyMs: totals.totalLatencyMs,
-      totalTokensIn: totals.totalTokensIn,
-      totalTokensOut: totals.totalTokensOut
-    };
   }
-  listSuites() {
-    const rows = this.#stmt(
-      "SELECT * FROM suites ORDER BY created_at DESC"
-    ).all();
-    return rows.map((row) => ({
-      id: row.id,
-      name: row.name,
-      created_at: row.created_at
-    }));
-  }
-  createPrompt(name, content) {
-    const id = crypto.randomUUID();
-    const now = Date.now();
-    const latest = this.#stmt(
-      "SELECT MAX(version) as latestVersion FROM prompts WHERE name = ?"
-    ).get(name);
-    const version = (latest?.latestVersion ?? 0) + 1;
-    this.#stmt(
-      "INSERT INTO prompts (id, name, version, content, created_at) VALUES (?, ?, ?, ?, ?)"
-    ).run(id, name, version, content, now);
-    return { id, name, version, content, created_at: now };
-  }
-  listPrompts() {
-    const rows = this.#stmt(
-      "SELECT * FROM prompts ORDER BY name COLLATE NOCASE ASC, version DESC"
-    ).all();
-    return rows.map((row) => ({
-      id: row.id,
-      name: row.name,
-      version: row.version,
-      content: row.content,
-      created_at: row.created_at
-    }));
-  }
-  getPrompt(id) {
-    const row = this.#stmt("SELECT * FROM prompts WHERE id = ?").get(id);
-    if (!row) return void 0;
-    return {
-      id: row.id,
-      name: row.name,
-      version: row.version,
-      content: row.content,
-      created_at: row.created_at
-    };
+  async #executeSingle() {
+    const options = this.#options;
+    let ds = options.dataset;
+    if (this.#selection.type === "failed") {
+      const indexes = resolveFailedIndexes(
+        options.store,
+        options.name,
+        options.model,
+        options.threshold
+      );
+      if (indexes.size > 0) {
+        ds = filterRecordsByIndex(ds, indexes);
+      }
+    } else {
+      ds = this.#applyDatasetFilter(ds);
+    }
+    const result = await evaluateSingle({ ...options, dataset: ds });
+    if (this.#shouldAssert && result.failCount > 0) {
+      throw new EvalAssertionError(result);
+    }
+    return result;
   }
-  deletePrompt(id) {
-    this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
+  async #executeMulti() {
+    const options = this.#options;
+    let result;
+    if (this.#selection.type === "failed") {
+      const perModelIndexes = /* @__PURE__ */ new Map();
+      for (const variant of options.models) {
+        perModelIndexes.set(
+          variant.name,
+          resolveFailedIndexes(
+            options.store,
+            options.name,
+            variant.name,
+            options.threshold
+          )
+        );
+      }
+      result = await evaluateEach(options, perModelIndexes);
+    } else {
+      const filtered = this.#applyDatasetFilter(options.dataset);
+      result = await evaluateEach({ ...options, dataset: filtered });
+    }
+    if (this.#shouldAssert && result.some((s) => s.failCount > 0)) {
+      throw new EvalAssertionError(result);
+    }
+    return result;
   }
 };
-
-// packages/evals/src/evaluate/index.ts
-async function evaluate(options) {
+function evaluate(options) {
   if ("models" in options) {
-    return evaluateEach(options);
+    return new EvalBuilder(options);
   }
-  return evaluateSingle(options);
-}
-function resolveStore(store) {
-  return store instanceof RunStore ? store : new RunStore(store);
+  return new EvalBuilder(options);
 }
 function wireReporters(reporters) {
   const emitter = new EvalEmitter();
@@ -918,7 +779,6 @@ async function notifyRunEnd(reporters, data) {
   await Promise.all(reporters.map((r) => r.onRunEnd?.(data)));
 }
 async function evaluateSingle(options) {
-  const store = resolveStore(options.store);
   const threshold = options.threshold ?? 0.5;
   const { emitter, cases, getRunId } = wireReporters(options.reporters);
   const summary = await runEval({
@@ -927,7 +787,7 @@ async function evaluateSingle(options) {
     dataset: options.dataset,
     task: options.task,
     scorers: options.scorers,
-    store,
+    store: options.store,
     emitter,
     suiteId: options.suiteId,
     maxConcurrency: options.maxConcurrency,
@@ -945,33 +805,39 @@ async function evaluateSingle(options) {
   });
   return summary;
 }
-async function evaluateEach(options) {
-  const store = resolveStore(options.store);
+async function evaluateEach(options, perModelFailedIndexes) {
   const items = [];
   for await (const item of options.dataset) {
     items.push(item);
   }
-  const suite = store.createSuite(options.name);
+  const suite = options.store.createSuite(options.name);
   return Promise.all(
-    options.models.map(
-      (variant) => evaluateSingle({
+    options.models.map((variant) => {
+      let ds = dataset(items);
+      const failedIndexes = perModelFailedIndexes?.get(variant.name);
+      if (failedIndexes && failedIndexes.size > 0) {
+        ds = filterRecordsByIndex(ds, failedIndexes);
+      }
+      return evaluateSingle({
         name: `${options.name} [${variant.name}]`,
         model: variant.name,
-        dataset: dataset(items),
+        dataset: ds,
         task: (input) => options.task(input, variant),
         scorers: options.scorers,
         reporters: options.reporters,
-        store,
+        store: options.store,
         suiteId: suite.id,
         maxConcurrency: options.maxConcurrency,
         timeout: options.timeout,
         trials: options.trials,
         threshold: options.threshold
-      })
-    )
+      });
+    })
   );
 }
 export {
+  EvalAssertionError,
+  EvalBuilder,
   evaluate
 };
 //# sourceMappingURL=index.js.map