@deepagents/evals 7.1.1 → 7.1.3

package/dist/index.js

@@ -363,6 +363,7 @@ function dataset(source) {
 
 // packages/evals/src/scorers/index.ts
 import OpenAI from "openai";
+import pMemoize from "p-memoize";
 var exactMatch = async ({ output, expected }) => {
   const exp = expected == null ? "" : String(expected);
   if (output === exp) return { score: 1 };
@@ -484,10 +485,14 @@ var LlmJudge = class _LlmJudge {
   #model;
   #choices;
   #criterion;
+  /** The configured client, or one OpenAI client made on first use. */
   #client;
   constructor(config, choices, criterion) {
     this.#model = config.model;
-    this.#client = config.client;
+    const configured = config.client;
+    this.#client = configured ? async () => configured : pMemoize(
+      async () => new OpenAI({ apiKey: process.env["OPENAI_API_KEY"] })
+    );
     this.#choices = choices;
     this.#criterion = criterion;
   }
@@ -520,10 +525,8 @@ var LlmJudge = class _LlmJudge {
     return (args) => this.#score(args);
   }
   async #score({ input, output, expected }) {
-    this.#client ??= new OpenAI({
-      apiKey: process.env["OPENAI_API_KEY"]
-    });
-    const response = await this.#client.chat.completions.create({
+    const client = await this.#client();
+    const response = await client.chat.completions.create({
       model: this.#model,
       messages: [
         {
@@ -645,6 +648,8 @@ function weighted(config) {
 }
 
 // packages/evals/src/store/index.ts
+import { Declaration } from "@zukhruf/sqlite-drift";
+import { transaction } from "@zukhruf/sqlite-transaction";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import {
@@ -652,9 +657,10 @@ import {
 } from "node:sqlite";
 
 // packages/evals/src/store/ddl.sqlite.sql
-var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
+var ddl_sqlite_default = "CREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  version INTEGER NOT NULL DEFAULT 1,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),\n  UNIQUE(name, version)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\nCREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC);\n";
 
 // packages/evals/src/store/index.ts
+var schema = new Declaration([ddl_sqlite_default], { rows: "keep" });
 var RUN_STATUSES = ["running", "completed", "failed"];
 function isRunStatus(value) {
   return RUN_STATUSES.some((status) => status === value);
@@ -802,20 +808,6 @@ var RunStore = class {
   #all(columns, sql2, ...params) {
     return this.#stmt(sql2).all(...params).map((row) => readRow(row, columns, sql2));
   }
-  #transaction(fn) {
-    this.#db.exec("BEGIN TRANSACTION");
-    try {
-      const result = fn();
-      this.#db.exec("COMMIT");
-      return result;
-    } catch (error) {
-      try {
-        this.#db.exec("ROLLBACK");
-      } catch {
-      }
-      throw error;
-    }
-  }
   constructor(pathOrDb) {
     if (pathOrDb instanceof DatabaseSync) {
       this.#db = pathOrDb;
@@ -824,93 +816,22 @@ var RunStore = class {
       mkdirSync(dirname(dbPath), { recursive: true });
       this.#db = new DatabaseSync(dbPath);
     }
-    this.#db.exec(ddl_sqlite_default);
-    this.#migrateRunsTableToSuiteRequired();
-    this.#migratePromptsTableIfNeeded();
-    this.#db.exec(
-      "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
+    this.#db.exec(`
+      PRAGMA journal_mode = WAL;
+      PRAGMA synchronous = NORMAL;
+      PRAGMA foreign_keys = ON;
+      PRAGMA busy_timeout = 5000;
+    `);
+    const runsTable = this.#get(
+      { name: "text" },
+      "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'runs'"
     );
-  }
-  #migratePromptsTableIfNeeded() {
-    const columns = this.#all({ name: "text" }, "PRAGMA table_info(prompts)");
-    if (columns.length === 0) return;
-    if (columns.some((column) => column.name === "version")) return;
-    this.#transaction(() => {
-      this.#db.exec("ALTER TABLE prompts RENAME TO prompts_legacy");
-      this.#db.exec(`
-        CREATE TABLE prompts (
-          id TEXT PRIMARY KEY,
-          name TEXT NOT NULL,
-          version INTEGER NOT NULL,
-          content TEXT NOT NULL,
-          created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
-          UNIQUE(name, version)
-        )
-      `);
-      this.#db.exec(`
-        INSERT INTO prompts (id, name, version, content, created_at)
-        SELECT id, name, 1, content, created_at
-        FROM prompts_legacy
-      `);
-      this.#db.exec("DROP TABLE prompts_legacy");
+    if (runsTable !== void 0) {
       this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at)"
+        "DELETE FROM runs WHERE suite_id IS NULL OR NOT EXISTS (SELECT 1 FROM suites WHERE suites.id = runs.suite_id)"
       );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
-      );
-    });
-  }
-  #migrateRunsTableToSuiteRequired() {
-    const columns = this.#all(
-      { name: "text", notnull: "number" },
-      "PRAGMA table_info(runs)"
-    );
-    if (columns.length === 0) return;
-    const suiteColumn = columns.find((column) => column.name === "suite_id");
-    const hasNonNullSuite = suiteColumn?.notnull === 1;
-    const runForeignKeys = this.#all(
-      { from: "text", on_delete: "text", table: "text" },
-      "PRAGMA foreign_key_list(runs)"
-    );
-    const suiteForeignKey = runForeignKeys.find(
-      (fk) => fk.from === "suite_id" && fk.table === "suites"
-    );
-    const hasCascadeDelete = suiteForeignKey?.on_delete === "CASCADE";
-    if (hasNonNullSuite && hasCascadeDelete) return;
-    this.#statements.clear();
-    this.#transaction(() => {
-      this.#db.exec(`
-        CREATE TABLE runs_next (
-          id TEXT PRIMARY KEY,
-          suite_id TEXT NOT NULL,
-          name TEXT NOT NULL,
-          model TEXT NOT NULL,
-          config TEXT,
-          started_at INTEGER NOT NULL,
-          finished_at INTEGER,
-          status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),
-          summary TEXT,
-          FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE
-        )
-      `);
-      this.#db.exec("DELETE FROM runs WHERE suite_id IS NULL");
-      this.#db.exec(`
-        INSERT INTO runs_next (id, suite_id, name, model, config, started_at, finished_at, status, summary)
-        SELECT r.id, r.suite_id, r.name, r.model, r.config, r.started_at, r.finished_at, r.status, r.summary
-        FROM runs r
-        JOIN suites s ON s.id = r.suite_id
-      `);
-      this.#db.exec("DROP TABLE runs");
-      this.#db.exec("ALTER TABLE runs_next RENAME TO runs");
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id)"
-      );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at)"
-      );
-    });
-    this.#statements.clear();
+    }
+    schema.apply(this.#db);
   }
   createSuite(name) {
     const id = crypto.randomUUID();
@@ -955,35 +876,43 @@ var RunStore = class {
     ).run(Date.now(), status, summary ? JSON.stringify(summary) : null, runId);
   }
   saveCases(cases) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
-      );
-      for (const c of cases) {
-        stmt.run(
-          c.id,
-          c.run_id,
-          c.idx,
-          JSON.stringify(c.input),
-          c.output,
-          c.expected != null ? JSON.stringify(c.expected) : null,
-          c.latency_ms,
-          c.tokens_in,
-          c.tokens_out,
-          c.error ?? null
+    transaction(
+      this.#db,
+      () => {
+        const stmt = this.#stmt(
+          "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
         );
-      }
-    });
+        for (const c of cases) {
+          stmt.run(
+            c.id,
+            c.run_id,
+            c.idx,
+            JSON.stringify(c.input),
+            c.output,
+            c.expected != null ? JSON.stringify(c.expected) : null,
+            c.latency_ms,
+            c.tokens_in,
+            c.tokens_out,
+            c.error ?? null
+          );
+        }
+      },
+      { mode: "deferred" }
+    );
   }
   saveScores(scores) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
-      );
-      for (const s of scores) {
-        stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
-      }
-    });
+    transaction(
+      this.#db,
+      () => {
+        const stmt = this.#stmt(
+          "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
+        );
+        for (const s of scores) {
+          stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
+        }
+      },
+      { mode: "deferred" }
+    );
   }
   getRun(runId) {
     const row = this.#get(runColumns, "SELECT * FROM runs WHERE id = ?", runId);
@@ -1151,12 +1080,16 @@ var RunStore = class {
     this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
   }
   resetRun(id) {
-    this.#transaction(() => {
-      this.#stmt("DELETE FROM cases WHERE run_id = ?").run(id);
-      this.#stmt(
-        "UPDATE runs SET status = ?, started_at = ?, finished_at = NULL, summary = NULL WHERE id = ?"
-      ).run("running", Date.now(), id);
-    });
+    transaction(
+      this.#db,
+      () => {
+        this.#stmt("DELETE FROM cases WHERE run_id = ?").run(id);
+        this.#stmt(
+          "UPDATE runs SET status = ?, started_at = ?, finished_at = NULL, summary = NULL WHERE id = ?"
+        ).run("running", Date.now(), id);
+      },
+      { mode: "deferred" }
+    );
   }
   deleteRun(id) {
     this.#stmt("DELETE FROM runs WHERE id = ?").run(id);
@@ -2184,45 +2117,44 @@ function resolveFailedIndexes(store, suiteName, model, threshold) {
   );
   return new Set(failingCases.map((c) => c.idx));
 }
-var EvalBuilder = class {
+var EVERY_CASE = { selection: { type: "all" }, shouldAssert: false };
+var EvalBuilder = class _EvalBuilder {
   #plan;
-  #selection = { type: "all" };
-  #shouldAssert = false;
-  constructor(plan) {
+  #run;
+  constructor(plan, run) {
     this.#plan = plan;
+    this.#run = run;
   }
-  #setSelection(selection) {
-    if (this.#selection.type !== "all") {
+  #select(selection) {
+    if (this.#run.selection.type !== "all") {
       throw new Error(
-        `Cannot combine .${this.#selection.type}() with .${selection.type}()`
+        `Cannot combine .${this.#run.selection.type}() with .${selection.type}()`
       );
     }
-    this.#selection = selection;
-    return this;
+    return new _EvalBuilder(this.#plan, { ...this.#run, selection });
   }
   failed() {
-    return this.#setSelection({ type: "failed" });
+    return this.#select({ type: "failed" });
   }
   cases(spec) {
     const { indexes } = parseRecordSelection(spec);
-    return this.#setSelection({ type: "cases", indexes });
+    return this.#select({ type: "cases", indexes });
   }
   sample(count) {
     if (count < 1) {
       throw new Error("Sample count must be >= 1");
     }
-    return this.#setSelection({ type: "sample", count });
+    return this.#select({ type: "sample", count });
   }
   assert() {
-    this.#shouldAssert = true;
-    return this;
+    return new _EvalBuilder(this.#plan, { ...this.#run, shouldAssert: true });
   }
   then(onfulfilled, onrejected) {
     return this.#execute().then(onfulfilled, onrejected);
   }
   async #execute() {
-    const result = await this.#plan.run(this.#selection);
-    if (this.#shouldAssert && this.#plan.hasFailures(result)) {
+    const result = await this.#plan.run(this.#run.selection);
+    if (this.#run.shouldAssert && this.#plan.hasFailures(result)) {
       throw new EvalAssertionError(result);
     }
     return result;
@@ -2288,9 +2220,9 @@ var EachModelEval = class {
 };
 function evaluate(options) {
   if ("models" in options) {
-    return new EvalBuilder(new EachModelEval(options));
+    return new EvalBuilder(new EachModelEval(options), EVERY_CASE);
   }
-  return new EvalBuilder(new SingleModelEval(options));
+  return new EvalBuilder(new SingleModelEval(options), EVERY_CASE);
 }
 function wireReporters(reporters) {
   const emitter = new EvalEmitter();