@botpress/evals 1.19.0-beta.9 → 2.0.0-alpha.1

package/dist/stores/local-eval-store.js

@@ -0,0 +1,595 @@
+// src/stores/local-eval-store.ts
+import { Database } from "bun:sqlite";
+import { existsSync as existsSync2, mkdirSync, readdirSync as readdirSync2, readFileSync, renameSync } from "fs";
+import { join } from "path";
+
+// src/loader.ts
+import { readdirSync, existsSync } from "fs";
+import { resolve } from "path";
+
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+
+// src/errors.ts
+var EvalRunnerError = class extends AdkError {
+};
+
+// src/loader.ts
+function isEvalDefinition(value) {
+  return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
+}
+async function loadEvalFile(filePath) {
+  const absPath = resolve(filePath);
+  let mod;
+  try {
+    mod = await import(absPath);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
+
+Make sure your eval file:
+  - Has no syntax or type errors
+  - Exports one or more \`new Eval({...})\` instances
+  - Has all dependencies installed (\`bun install\`)`,
+      expected: true,
+      cause: err
+    });
+  }
+  const results = [];
+  for (const [key, value] of Object.entries(mod)) {
+    if (key === "__esModule") continue;
+    if (value instanceof Eval || isEvalDefinition(value)) {
+      results.push(value);
+    }
+  }
+  if (results.length === 0) {
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
+  }
+  return results;
+}
+async function loadEvalsFromDir(dirPath) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) {
+    return [];
+  }
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  const evals = [];
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    evals.push(...defs);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const e of evals) {
+    if (seen.has(e.name)) {
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
+    }
+    seen.add(e.name);
+  }
+  return evals;
+}
+function filterEvals(evals, filter) {
+  if (!filter) return evals;
+  return evals.filter((e) => {
+    if (filter.names && filter.names.length > 0) {
+      if (!filter.names.includes(e.name)) return false;
+    }
+    if (filter.tags && filter.tags.length > 0) {
+      if (!e.tags || !filter.tags.some((t) => e.tags.includes(t))) return false;
+    }
+    if (filter.type) {
+      if (e.type !== filter.type) return false;
+    }
+    return true;
+  });
+}
+
+// src/stores/eval-definition-loader.ts
+function createDiskEvalLoader(evalsDir) {
+  return () => loadEvalsFromDir(evalsDir);
+}
+
+// src/stores/local-eval-store.ts
+var errText = (err) => err instanceof Error ? err.message : String(err);
+var SCHEMA_VERSION = "2";
+var SCHEMA = `
+CREATE TABLE IF NOT EXISTS eval_runs (
+  id              TEXT PRIMARY KEY,
+  timestamp       TEXT NOT NULL,
+  started_at      INTEGER NOT NULL,
+  passed          INTEGER NOT NULL,
+  failed          INTEGER NOT NULL,
+  total           INTEGER NOT NULL,
+  duration        REAL NOT NULL,
+  bot_duration    REAL NOT NULL DEFAULT 0,
+  eval_duration   REAL NOT NULL DEFAULT 0,
+  aborted         INTEGER NOT NULL DEFAULT 0,
+  filter_json     TEXT,
+  eval_names      TEXT NOT NULL DEFAULT '[]'
+);
+CREATE INDEX IF NOT EXISTS idx_eval_runs_started ON eval_runs(started_at DESC);
+
+CREATE TABLE IF NOT EXISTS eval_reports (
+  run_id          TEXT NOT NULL REFERENCES eval_runs(id) ON DELETE CASCADE,
+  seq             INTEGER NOT NULL,
+  name            TEXT NOT NULL,
+  started_at      INTEGER NOT NULL,
+  pass            INTEGER NOT NULL,
+  duration        REAL NOT NULL,
+  type            TEXT,
+  error           TEXT,
+  report_json     TEXT NOT NULL,
+  PRIMARY KEY (run_id, seq)
+);
+CREATE INDEX IF NOT EXISTS idx_eval_reports_name_time ON eval_reports(name, started_at DESC);
+
+CREATE TABLE IF NOT EXISTS schema_meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+`;
+function aggregateTurnTimings(report) {
+  let botDuration = 0;
+  let evalDuration = 0;
+  for (const e of report.evals) {
+    for (const t of e.turns) {
+      botDuration += t.botDuration;
+      evalDuration += t.evalDuration;
+    }
+  }
+  return { botDuration, evalDuration };
+}
+function toEvalSummary(e) {
+  return {
+    name: e.name,
+    description: e.description,
+    tags: e.tags || [],
+    type: e.type || "capability",
+    turnCount: e.conversation.length,
+    hasOutcome: !!e.outcome
+  };
+}
+var LocalEvalStore = class {
+  db = null;
+  dbPath;
+  evalsDir;
+  insertRunStmt = null;
+  insertReportStmt = null;
+  _getActiveRunId;
+  _loadEvalDefinitions;
+  constructor(config) {
+    this.evalsDir = join(config.agentPath, ".adk", "evals");
+    this.dbPath = join(this.evalsDir, "evals.db");
+    this._getActiveRunId = config.getActiveRunId ?? (() => null);
+    this._loadEvalDefinitions = config.loadEvalDefinitions ?? createDiskEvalLoader(join(config.agentPath, "evals"));
+  }
+  // ── Eval definitions (from disk) ───────────────────────────────────
+  async listEvals(filter) {
+    const allEvals = await this._loadEvalDefinitions();
+    const evals = filter ? filterEvals(allEvals, filter) : allEvals;
+    return evals.map(toEvalSummary);
+  }
+  async getEval(name) {
+    const allEvals = await this._loadEvalDefinitions();
+    return allEvals.find((e) => e.name === name) ?? null;
+  }
+  // ── Run lifecycle — write ───────────────────────────────────────────
+  async createRun(_runType, _metadata) {
+    return crypto.randomUUID().replace(/-/g, "").slice(0, 26);
+  }
+  async addRunResults(_runId, _evalReport) {
+  }
+  async completeRun(_runId, report) {
+    const db = this.getDb();
+    if (!db) return;
+    db.run("BEGIN");
+    try {
+      this.insertRunReport(report);
+      db.run("COMMIT");
+    } catch (err) {
+      db.run("ROLLBACK");
+      console.error(`[local-eval-store] completeRun transaction failed (run ${report.id}): ${errText(err)}`);
+      throw err;
+    }
+  }
+  // ── Run results — read ─────────────────────────────────────────────
+  async loadRunResult(runId) {
+    const db = this.getDb();
+    if (!db) return null;
+    const runRow = db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(runId);
+    if (!runRow) return null;
+    const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
+    return this.assembleRun(runRow, reportRows);
+  }
+  async getLatestRun() {
+    const db = this.getDb();
+    if (!db) return null;
+    const runRow = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT 1").get();
+    if (!runRow) return null;
+    const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
+    return this.assembleRun(runRow, reportRows);
+  }
+  async listRunSummaries(opts) {
+    const db = this.getDb();
+    if (!db) return [];
+    const limit = opts?.limit ?? 50;
+    const sinceTs = opts?.since;
+    const rows = sinceTs !== void 0 ? db.prepare(
+      "SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?"
+    ).all(sinceTs, limit) : db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
+    return rows.map((r) => ({
+      id: r.id,
+      timestamp: r.timestamp,
+      passed: r.passed,
+      failed: r.failed,
+      total: r.total,
+      duration: r.duration,
+      botDuration: r.bot_duration,
+      evalDuration: r.eval_duration,
+      filter: r.filter_json ? JSON.parse(r.filter_json) : void 0,
+      evalNames: JSON.parse(r.eval_names),
+      aborted: r.aborted === 1
+    }));
+  }
+  async listEvalReportsByName(evalName, opts) {
+    const db = this.getDb();
+    if (!db) return [];
+    const limit = opts?.limit ?? 20;
+    const sinceTs = opts?.since;
+    const baseSql = `
+      SELECT er.*,
+             r.timestamp    AS run_timestamp,
+             r.total        AS run_total,
+             r.filter_json  AS run_filter_json
+      FROM eval_reports er
+      JOIN eval_runs r ON r.id = er.run_id
+      WHERE er.name = ?
+    `;
+    const rows = sinceTs !== void 0 ? db.prepare(
+      `${baseSql} AND er.started_at >= ? ORDER BY er.started_at DESC LIMIT ?`
+    ).all(evalName, sinceTs, limit) : db.prepare(`${baseSql} ORDER BY er.started_at DESC LIMIT ?`).all(evalName, limit);
+    return rows.map((row) => ({
+      runId: row.run_id,
+      timestamp: row.run_timestamp,
+      report: JSON.parse(row.report_json),
+      totalEvalsInRun: row.run_total,
+      filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : void 0
+    }));
+  }
+  async listEvalReportsBulk(opts) {
+    const db = this.getDb();
+    if (!db) return {};
+    const perEval = opts?.perEval ?? 20;
+    const sinceTs = opts?.since;
+    const sinceClause = sinceTs !== void 0 ? "WHERE er.started_at >= ?" : "";
+    const sql = `
+      WITH ranked AS (
+        SELECT er.*, ROW_NUMBER() OVER (PARTITION BY name ORDER BY started_at DESC) AS rn
+        FROM eval_reports er
+        ${sinceClause}
+      )
+      SELECT ranked.run_id, ranked.name, ranked.started_at, ranked.pass,
+             ranked.duration, ranked.type, ranked.error, ranked.report_json,
+             r.timestamp   AS run_timestamp,
+             r.total       AS run_total,
+             r.filter_json AS run_filter_json
+      FROM ranked
+      JOIN eval_runs r ON r.id = ranked.run_id
+      WHERE ranked.rn <= ?
+      ORDER BY ranked.name, ranked.started_at DESC
+    `;
+    const rows = sinceTs !== void 0 ? db.prepare(sql).all(sinceTs, perEval) : db.prepare(sql).all(perEval);
+    const out = {};
+    for (const row of rows) {
+      const entry = {
+        runId: row.run_id,
+        timestamp: row.run_timestamp,
+        report: JSON.parse(row.report_json),
+        totalEvalsInRun: row.run_total,
+        filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : void 0
+      };
+      const arr = out[row.name];
+      if (arr) arr.push(entry);
+      else out[row.name] = [entry];
+    }
+    return out;
+  }
+  // ── Live run tracking ──────────────────────────────────────────────
+  async *watchRun(_signal, _options) {
+  }
+  // ── Runner state ───────────────────────────────────────────────────
+  async getRunnerState() {
+    const runId = this._getActiveRunId();
+    return { running: runId !== null, runId };
+  }
+  // ── Extra (not on EvalStore interface) ──────────────────────────────
+  async listRunResults(limit = 50, sinceTs) {
+    const db = this.getDb();
+    if (!db) return [];
+    let runRows;
+    if (sinceTs !== void 0) {
+      runRows = db.prepare(
+        "SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?"
+      ).all(sinceTs, limit);
+    } else {
+      runRows = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
+    }
+    if (runRows.length === 0) return [];
+    const ids = runRows.map((r) => r.id);
+    const placeholders = ids.map(() => "?").join(",");
+    const reportRows = db.prepare(`SELECT * FROM eval_reports WHERE run_id IN (${placeholders}) ORDER BY seq`).all(...ids);
+    const byRun = /* @__PURE__ */ new Map();
+    for (const row of reportRows) {
+      const arr = byRun.get(row.run_id);
+      if (arr) arr.push(row);
+      else byRun.set(row.run_id, [row]);
+    }
+    return runRows.map((row) => this.assembleRun(row, byRun.get(row.id) ?? []));
+  }
+  // ── Lifecycle ──────────────────────────────────────────────────────
+  close() {
+    try {
+      this.db?.close();
+    } catch {
+    }
+    this.db = null;
+    this.insertRunStmt = null;
+    this.insertReportStmt = null;
+  }
+  getDbPath() {
+    return this.dbPath;
+  }
+  // ── SQLite internals ───────────────────────────────────────────────
+  getDb() {
+    if (this.db) return this.db;
+    mkdirSync(this.evalsDir, { recursive: true });
+    try {
+      this.db = new Database(this.dbPath);
+      this.db.run("PRAGMA journal_mode=WAL");
+      this.db.run("PRAGMA synchronous=NORMAL");
+      this.db.run("PRAGMA busy_timeout=5000");
+      this.db.run("PRAGMA foreign_keys=ON");
+      this.db.run(SCHEMA);
+      this.setMeta("schema_version", SCHEMA_VERSION);
+      this.insertRunStmt = this.db.prepare(`
+        INSERT OR REPLACE INTO eval_runs (
+          id, timestamp, started_at, passed, failed, total, duration,
+          bot_duration, eval_duration, aborted, filter_json, eval_names
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+      this.insertReportStmt = this.db.prepare(`
+        INSERT OR REPLACE INTO eval_reports (
+          run_id, seq, name, started_at, pass, duration, type, error, report_json
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+      this.importLegacyJsonRunsIfNeeded();
+      return this.db;
+    } catch (err) {
+      console.warn("[local-eval-store] Failed to initialize database:", err);
+      try {
+        this.db?.close();
+      } catch {
+      }
+      try {
+        if (existsSync2(this.dbPath)) {
+          renameSync(this.dbPath, `${this.dbPath}.corrupt-${Date.now()}`);
+        }
+      } catch {
+      }
+      this.db = null;
+      this.insertRunStmt = null;
+      this.insertReportStmt = null;
+      return null;
+    }
+  }
+  getMeta(key) {
+    const db = this.db;
+    if (!db) return null;
+    const row = db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(key);
+    return row?.value ?? null;
+  }
+  setMeta(key, value) {
+    const db = this.db;
+    if (!db) return;
+    db.prepare("INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)").run(key, value);
+  }
+  importLegacyJsonRunsIfNeeded() {
+    const db = this.db;
+    if (!db) return;
+    if (this.getMeta("imported_legacy") === "1") return;
+    const legacyDir = join(this.evalsDir, "runs");
+    if (!existsSync2(legacyDir)) {
+      this.setMeta("imported_legacy", "1");
+      return;
+    }
+    let files = [];
+    try {
+      files = readdirSync2(legacyDir).filter((f) => f.endsWith(".json"));
+    } catch (err) {
+      console.warn("[local-eval-store] Could not read legacy runs dir:", err);
+      return;
+    }
+    if (files.length === 0) {
+      this.setMeta("imported_legacy", "1");
+      return;
+    }
+    let imported = 0;
+    let skipped = 0;
+    db.run("BEGIN");
+    try {
+      for (const file of files) {
+        db.run("SAVEPOINT import_file");
+        try {
+          const raw = readFileSync(join(legacyDir, file), "utf-8");
+          const report = JSON.parse(raw);
+          this.insertRunReport(report);
+          db.run("RELEASE import_file");
+          imported++;
+        } catch (err) {
+          db.run("ROLLBACK TO import_file");
+          db.run("RELEASE import_file");
+          skipped++;
+          console.warn(`[local-eval-store] Skipping unreadable legacy run ${file}:`, err);
+        }
+      }
+      this.setMeta("imported_legacy", "1");
+      db.run("COMMIT");
+    } catch (err) {
+      db.run("ROLLBACK");
+      console.warn("[local-eval-store] Legacy import failed, will retry on next start:", err);
+      return;
+    }
+    try {
+      renameSync(legacyDir, join(this.evalsDir, "runs.json-migrated"));
+    } catch (err) {
+      console.warn("[local-eval-store] Could not rename legacy runs dir:", err);
+    }
+    console.log(
+      `[local-eval-store] Imported ${imported} legacy eval run${imported === 1 ? "" : "s"}` + (skipped > 0 ? ` (${skipped} skipped)` : "")
+    );
+  }
+  insertRunReport(report) {
+    if (!this.insertRunStmt || !this.insertReportStmt) return;
+    const startedAt = Date.parse(report.timestamp);
+    const { botDuration, evalDuration } = aggregateTurnTimings(report);
+    this.insertRunStmt.run(
+      report.id,
+      report.timestamp,
+      Number.isNaN(startedAt) ? 0 : startedAt,
+      report.passed,
+      report.failed,
+      report.total,
+      report.duration,
+      botDuration,
+      evalDuration,
+      report.aborted ? 1 : 0,
+      report.filter ? JSON.stringify(report.filter) : null,
+      JSON.stringify(report.evals.map((e) => e.name))
+    );
+    for (let i = 0; i < report.evals.length; i++) {
+      const evalReport = report.evals[i];
+      this.insertReportStmt.run(
+        report.id,
+        i,
+        evalReport.name,
+        Number.isNaN(startedAt) ? 0 : startedAt,
+        evalReport.pass ? 1 : 0,
+        evalReport.duration,
+        evalReport.type ?? null,
+        evalReport.error ?? null,
+        JSON.stringify(evalReport)
+      );
+    }
+  }
+  assembleRun(runRow, reportRows) {
+    const evals = reportRows.map((r) => JSON.parse(r.report_json));
+    const report = {
+      id: runRow.id,
+      timestamp: runRow.timestamp,
+      evals,
+      passed: runRow.passed,
+      failed: runRow.failed,
+      total: runRow.total,
+      duration: runRow.duration
+    };
+    if (runRow.filter_json) {
+      report.filter = JSON.parse(runRow.filter_json);
+    }
+    if (runRow.aborted === 1) {
+      report.aborted = true;
+    }
+    return report;
+  }
+};
+var stores = /* @__PURE__ */ new Map();
+function getLocalEvalStore(agentPath, getActiveRunId) {
+  let store = stores.get(agentPath);
+  if (!store) {
+    store = new LocalEvalStore({ agentPath, getActiveRunId });
+    stores.set(agentPath, store);
+  }
+  return store;
+}
+function closeLocalEvalStores(agentPath) {
+  if (agentPath) {
+    stores.get(agentPath)?.close();
+    stores.delete(agentPath);
+    return;
+  }
+  for (const store of stores.values()) store.close();
+  stores.clear();
+}
+export {
+  LocalEvalStore,
+  closeLocalEvalStores,
+  getLocalEvalStore
+};
+//# sourceMappingURL=local-eval-store.js.map