@deepagents/evals 7.1.2 → 7.1.4

package/dist/index.js

@@ -648,6 +648,8 @@ function weighted(config) {
 }
 
 // packages/evals/src/store/index.ts
+import { Declaration } from "@zukhruf/sqlite-drift";
+import { transaction } from "@zukhruf/sqlite-transaction";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import {
@@ -655,9 +657,10 @@ import {
 } from "node:sqlite";
 
 // packages/evals/src/store/ddl.sqlite.sql
-var ddl_sqlite_default = "PRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA foreign_keys = ON;\n\nCREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL UNIQUE,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\n";
+var ddl_sqlite_default = "CREATE TABLE IF NOT EXISTS suites (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n);\n\nCREATE TABLE IF NOT EXISTS runs (\n  id TEXT PRIMARY KEY,\n  suite_id TEXT NOT NULL,\n  name TEXT NOT NULL,\n  model TEXT NOT NULL,\n  config TEXT,\n  started_at INTEGER NOT NULL,\n  finished_at INTEGER,\n  status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),\n  summary TEXT,\n  FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id);\nCREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at);\n\nCREATE TABLE IF NOT EXISTS cases (\n  id TEXT PRIMARY KEY,\n  run_id TEXT NOT NULL,\n  idx INTEGER NOT NULL,\n  input TEXT NOT NULL,\n  output TEXT,\n  expected TEXT,\n  latency_ms INTEGER,\n  tokens_in INTEGER,\n  tokens_out INTEGER,\n  error TEXT,\n  FOREIGN KEY (run_id) REFERENCES runs(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_cases_run_id ON cases(run_id);\n\nCREATE TABLE IF NOT EXISTS scores (\n  id TEXT PRIMARY KEY,\n  case_id TEXT NOT NULL,\n  scorer_name TEXT NOT NULL,\n  score REAL NOT NULL,\n  reason TEXT,\n  FOREIGN KEY (case_id) REFERENCES cases(id) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS idx_scores_case_id ON scores(case_id);\n\nCREATE TABLE IF NOT EXISTS prompts (\n  id TEXT PRIMARY KEY,\n  name TEXT NOT NULL,\n  version INTEGER NOT NULL DEFAULT 1,\n  content TEXT NOT NULL,\n  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),\n  UNIQUE(name, version)\n);\n\nCREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at);\nCREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC);\n";
 
 // packages/evals/src/store/index.ts
+var schema = new Declaration([ddl_sqlite_default], { rows: "keep" });
 var RUN_STATUSES = ["running", "completed", "failed"];
 function isRunStatus(value) {
   return RUN_STATUSES.some((status) => status === value);
@@ -805,20 +808,6 @@ var RunStore = class {
   #all(columns, sql2, ...params) {
     return this.#stmt(sql2).all(...params).map((row) => readRow(row, columns, sql2));
   }
-  #transaction(fn) {
-    this.#db.exec("BEGIN TRANSACTION");
-    try {
-      const result = fn();
-      this.#db.exec("COMMIT");
-      return result;
-    } catch (error) {
-      try {
-        this.#db.exec("ROLLBACK");
-      } catch {
-      }
-      throw error;
-    }
-  }
   constructor(pathOrDb) {
     if (pathOrDb instanceof DatabaseSync) {
       this.#db = pathOrDb;
@@ -827,93 +816,22 @@ var RunStore = class {
       mkdirSync(dirname(dbPath), { recursive: true });
       this.#db = new DatabaseSync(dbPath);
     }
-    this.#db.exec(ddl_sqlite_default);
-    this.#migrateRunsTableToSuiteRequired();
-    this.#migratePromptsTableIfNeeded();
-    this.#db.exec(
-      "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
-    );
-  }
-  #migratePromptsTableIfNeeded() {
-    const columns = this.#all({ name: "text" }, "PRAGMA table_info(prompts)");
-    if (columns.length === 0) return;
-    if (columns.some((column) => column.name === "version")) return;
-    this.#transaction(() => {
-      this.#db.exec("ALTER TABLE prompts RENAME TO prompts_legacy");
-      this.#db.exec(`
-        CREATE TABLE prompts (
-          id TEXT PRIMARY KEY,
-          name TEXT NOT NULL,
-          version INTEGER NOT NULL,
-          content TEXT NOT NULL,
-          created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
-          UNIQUE(name, version)
-        )
-      `);
-      this.#db.exec(`
-        INSERT INTO prompts (id, name, version, content, created_at)
-        SELECT id, name, 1, content, created_at
-        FROM prompts_legacy
-      `);
-      this.#db.exec("DROP TABLE prompts_legacy");
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_created_at ON prompts(created_at)"
-      );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_prompts_name_version ON prompts(name, version DESC)"
-      );
-    });
-  }
-  #migrateRunsTableToSuiteRequired() {
-    const columns = this.#all(
-      { name: "text", notnull: "number" },
-      "PRAGMA table_info(runs)"
-    );
-    if (columns.length === 0) return;
-    const suiteColumn = columns.find((column) => column.name === "suite_id");
-    const hasNonNullSuite = suiteColumn?.notnull === 1;
-    const runForeignKeys = this.#all(
-      { from: "text", on_delete: "text", table: "text" },
-      "PRAGMA foreign_key_list(runs)"
-    );
-    const suiteForeignKey = runForeignKeys.find(
-      (fk) => fk.from === "suite_id" && fk.table === "suites"
+    this.#db.exec(`
+      PRAGMA journal_mode = WAL;
+      PRAGMA synchronous = NORMAL;
+      PRAGMA foreign_keys = ON;
+      PRAGMA busy_timeout = 5000;
+    `);
+    const runsTable = this.#get(
+      { name: "text" },
+      "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'runs'"
     );
-    const hasCascadeDelete = suiteForeignKey?.on_delete === "CASCADE";
-    if (hasNonNullSuite && hasCascadeDelete) return;
-    this.#statements.clear();
-    this.#transaction(() => {
-      this.#db.exec(`
-        CREATE TABLE runs_next (
-          id TEXT PRIMARY KEY,
-          suite_id TEXT NOT NULL,
-          name TEXT NOT NULL,
-          model TEXT NOT NULL,
-          config TEXT,
-          started_at INTEGER NOT NULL,
-          finished_at INTEGER,
-          status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running', 'completed', 'failed')),
-          summary TEXT,
-          FOREIGN KEY (suite_id) REFERENCES suites(id) ON DELETE CASCADE
-        )
-      `);
-      this.#db.exec("DELETE FROM runs WHERE suite_id IS NULL");
-      this.#db.exec(`
-        INSERT INTO runs_next (id, suite_id, name, model, config, started_at, finished_at, status, summary)
-        SELECT r.id, r.suite_id, r.name, r.model, r.config, r.started_at, r.finished_at, r.status, r.summary
-        FROM runs r
-        JOIN suites s ON s.id = r.suite_id
-      `);
-      this.#db.exec("DROP TABLE runs");
-      this.#db.exec("ALTER TABLE runs_next RENAME TO runs");
+    if (runsTable !== void 0) {
       this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_suite_id ON runs(suite_id)"
+        "DELETE FROM runs WHERE suite_id IS NULL OR NOT EXISTS (SELECT 1 FROM suites WHERE suites.id = runs.suite_id)"
       );
-      this.#db.exec(
-        "CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at)"
-      );
-    });
-    this.#statements.clear();
+    }
+    schema.apply(this.#db);
   }
   createSuite(name) {
     const id = crypto.randomUUID();
@@ -958,35 +876,43 @@ var RunStore = class {
     ).run(Date.now(), status, summary ? JSON.stringify(summary) : null, runId);
   }
   saveCases(cases) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
-      );
-      for (const c of cases) {
-        stmt.run(
-          c.id,
-          c.run_id,
-          c.idx,
-          JSON.stringify(c.input),
-          c.output,
-          c.expected != null ? JSON.stringify(c.expected) : null,
-          c.latency_ms,
-          c.tokens_in,
-          c.tokens_out,
-          c.error ?? null
+    transaction(
+      this.#db,
+      () => {
+        const stmt = this.#stmt(
+          "INSERT INTO cases (id, run_id, idx, input, output, expected, latency_ms, tokens_in, tokens_out, error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
         );
-      }
-    });
+        for (const c of cases) {
+          stmt.run(
+            c.id,
+            c.run_id,
+            c.idx,
+            JSON.stringify(c.input),
+            c.output,
+            c.expected != null ? JSON.stringify(c.expected) : null,
+            c.latency_ms,
+            c.tokens_in,
+            c.tokens_out,
+            c.error ?? null
+          );
+        }
+      },
+      { mode: "deferred" }
+    );
   }
   saveScores(scores) {
-    this.#transaction(() => {
-      const stmt = this.#stmt(
-        "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
-      );
-      for (const s of scores) {
-        stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
-      }
-    });
+    transaction(
+      this.#db,
+      () => {
+        const stmt = this.#stmt(
+          "INSERT INTO scores (id, case_id, scorer_name, score, reason) VALUES (?, ?, ?, ?, ?)"
+        );
+        for (const s of scores) {
+          stmt.run(s.id, s.case_id, s.scorer_name, s.score, s.reason ?? null);
+        }
+      },
+      { mode: "deferred" }
+    );
   }
   getRun(runId) {
     const row = this.#get(runColumns, "SELECT * FROM runs WHERE id = ?", runId);
@@ -1154,12 +1080,16 @@ var RunStore = class {
     this.#stmt("DELETE FROM prompts WHERE id = ?").run(id);
   }
   resetRun(id) {
-    this.#transaction(() => {
-      this.#stmt("DELETE FROM cases WHERE run_id = ?").run(id);
-      this.#stmt(
-        "UPDATE runs SET status = ?, started_at = ?, finished_at = NULL, summary = NULL WHERE id = ?"
-      ).run("running", Date.now(), id);
-    });
+    transaction(
+      this.#db,
+      () => {
+        this.#stmt("DELETE FROM cases WHERE run_id = ?").run(id);
+        this.#stmt(
+          "UPDATE runs SET status = ?, started_at = ?, finished_at = NULL, summary = NULL WHERE id = ?"
+        ).run("running", Date.now(), id);
+      },
+      { mode: "deferred" }
+    );
   }
   deleteRun(id) {
     this.#stmt("DELETE FROM runs WHERE id = ?").run(id);