@botpress/evals 1.19.0-beta.7 → 2.0.0-alpha.0

package/dist/stores/index.js

@@ -0,0 +1,1118 @@
+// src/stores/local-eval-store.ts
+import { Database } from "bun:sqlite";
+import { existsSync as existsSync2, mkdirSync, readdirSync as readdirSync2, readFileSync, renameSync } from "fs";
+import { join } from "path";
+
+// src/loader.ts
+import { readdirSync, existsSync } from "fs";
+import { resolve } from "path";
+
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/errors.ts
+import { AdkError } from "@botpress/analytics";
+var EvalRunnerError = class extends AdkError {
+};
+
+// src/loader.ts
+function isEvalDefinition(value) {
+  return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
+}
+async function loadEvalFile(filePath) {
+  const absPath = resolve(filePath);
+  let mod;
+  try {
+    mod = await import(absPath);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
+
+Make sure your eval file:
+  - Has no syntax or type errors
+  - Exports one or more \`new Eval({...})\` instances
+  - Has all dependencies installed (\`bun install\`)`,
+      expected: true,
+      cause: err
+    });
+  }
+  const results = [];
+  for (const [key, value] of Object.entries(mod)) {
+    if (key === "__esModule") continue;
+    if (value instanceof Eval || isEvalDefinition(value)) {
+      results.push(value);
+    }
+  }
+  if (results.length === 0) {
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
+  }
+  return results;
+}
+async function loadEvalsFromDir(dirPath) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) {
+    return [];
+  }
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  const evals = [];
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    evals.push(...defs);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const e of evals) {
+    if (seen.has(e.name)) {
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
+    }
+    seen.add(e.name);
+  }
+  return evals;
+}
+function filterEvals(evals, filter) {
+  if (!filter) return evals;
+  return evals.filter((e) => {
+    if (filter.names && filter.names.length > 0) {
+      if (!filter.names.includes(e.name)) return false;
+    }
+    if (filter.tags && filter.tags.length > 0) {
+      if (!e.tags || !filter.tags.some((t) => e.tags.includes(t))) return false;
+    }
+    if (filter.type) {
+      if (e.type !== filter.type) return false;
+    }
+    return true;
+  });
+}
+
+// src/stores/eval-definition-loader.ts
+function createDiskEvalLoader(evalsDir) {
+  return () => loadEvalsFromDir(evalsDir);
+}
+
+// src/stores/local-eval-store.ts
+var errText = (err) => err instanceof Error ? err.message : String(err);
+var SCHEMA_VERSION = "2";
+var SCHEMA = `
+CREATE TABLE IF NOT EXISTS eval_runs (
+  id              TEXT PRIMARY KEY,
+  timestamp       TEXT NOT NULL,
+  started_at      INTEGER NOT NULL,
+  passed          INTEGER NOT NULL,
+  failed          INTEGER NOT NULL,
+  total           INTEGER NOT NULL,
+  duration        REAL NOT NULL,
+  bot_duration    REAL NOT NULL DEFAULT 0,
+  eval_duration   REAL NOT NULL DEFAULT 0,
+  aborted         INTEGER NOT NULL DEFAULT 0,
+  filter_json     TEXT,
+  eval_names      TEXT NOT NULL DEFAULT '[]'
+);
+CREATE INDEX IF NOT EXISTS idx_eval_runs_started ON eval_runs(started_at DESC);
+
+CREATE TABLE IF NOT EXISTS eval_reports (
+  run_id          TEXT NOT NULL REFERENCES eval_runs(id) ON DELETE CASCADE,
+  seq             INTEGER NOT NULL,
+  name            TEXT NOT NULL,
+  started_at      INTEGER NOT NULL,
+  pass            INTEGER NOT NULL,
+  duration        REAL NOT NULL,
+  type            TEXT,
+  error           TEXT,
+  report_json     TEXT NOT NULL,
+  PRIMARY KEY (run_id, seq)
+);
+CREATE INDEX IF NOT EXISTS idx_eval_reports_name_time ON eval_reports(name, started_at DESC);
+
+CREATE TABLE IF NOT EXISTS schema_meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+`;
+function aggregateTurnTimings(report) {
+  let botDuration = 0;
+  let evalDuration = 0;
+  for (const e of report.evals) {
+    for (const t of e.turns) {
+      botDuration += t.botDuration;
+      evalDuration += t.evalDuration;
+    }
+  }
+  return { botDuration, evalDuration };
+}
+function toEvalSummary(e) {
+  return {
+    name: e.name,
+    description: e.description,
+    tags: e.tags || [],
+    type: e.type || "capability",
+    turnCount: e.conversation.length,
+    hasOutcome: !!e.outcome
+  };
+}
+var LocalEvalStore = class {
+  db = null;
+  dbPath;
+  evalsDir;
+  insertRunStmt = null;
+  insertReportStmt = null;
+  _getActiveRunId;
+  _loadEvalDefinitions;
+  constructor(config) {
+    this.evalsDir = join(config.agentPath, ".adk", "evals");
+    this.dbPath = join(this.evalsDir, "evals.db");
+    this._getActiveRunId = config.getActiveRunId ?? (() => null);
+    this._loadEvalDefinitions = config.loadEvalDefinitions ?? createDiskEvalLoader(join(config.agentPath, "evals"));
+  }
+  // ── Eval definitions (from disk) ───────────────────────────────────
+  async listEvals(filter) {
+    const allEvals = await this._loadEvalDefinitions();
+    const evals = filter ? filterEvals(allEvals, filter) : allEvals;
+    return evals.map(toEvalSummary);
+  }
+  async getEval(name) {
+    const allEvals = await this._loadEvalDefinitions();
+    return allEvals.find((e) => e.name === name) ?? null;
+  }
+  // ── Run lifecycle — write ───────────────────────────────────────────
+  async createRun(_runType, _metadata) {
+    return crypto.randomUUID().replace(/-/g, "").slice(0, 26);
+  }
+  async addRunResults(_runId, _evalReport) {
+  }
+  async completeRun(_runId, report) {
+    const db = this.getDb();
+    if (!db) return;
+    db.run("BEGIN");
+    try {
+      this.insertRunReport(report);
+      db.run("COMMIT");
+    } catch (err) {
+      db.run("ROLLBACK");
+      console.error(`[local-eval-store] completeRun transaction failed (run ${report.id}): ${errText(err)}`);
+      throw err;
+    }
+  }
+  // ── Run results — read ─────────────────────────────────────────────
+  async loadRunResult(runId) {
+    const db = this.getDb();
+    if (!db) return null;
+    const runRow = db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(runId);
+    if (!runRow) return null;
+    const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
+    return this.assembleRun(runRow, reportRows);
+  }
+  async getLatestRun() {
+    const db = this.getDb();
+    if (!db) return null;
+    const runRow = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT 1").get();
+    if (!runRow) return null;
+    const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
+    return this.assembleRun(runRow, reportRows);
+  }
+  async listRunSummaries(opts) {
+    const db = this.getDb();
+    if (!db) return [];
+    const limit = opts?.limit ?? 50;
+    const sinceTs = opts?.since;
+    const rows = sinceTs !== void 0 ? db.prepare(
+      "SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?"
+    ).all(sinceTs, limit) : db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
+    return rows.map((r) => ({
+      id: r.id,
+      timestamp: r.timestamp,
+      passed: r.passed,
+      failed: r.failed,
+      total: r.total,
+      duration: r.duration,
+      botDuration: r.bot_duration,
+      evalDuration: r.eval_duration,
+      filter: r.filter_json ? JSON.parse(r.filter_json) : void 0,
+      evalNames: JSON.parse(r.eval_names),
+      aborted: r.aborted === 1
+    }));
+  }
+  async listEvalReportsByName(evalName, opts) {
+    const db = this.getDb();
+    if (!db) return [];
+    const limit = opts?.limit ?? 20;
+    const sinceTs = opts?.since;
+    const baseSql = `
+      SELECT er.*,
+             r.timestamp    AS run_timestamp,
+             r.total        AS run_total,
+             r.filter_json  AS run_filter_json
+      FROM eval_reports er
+      JOIN eval_runs r ON r.id = er.run_id
+      WHERE er.name = ?
+    `;
+    const rows = sinceTs !== void 0 ? db.prepare(
+      `${baseSql} AND er.started_at >= ? ORDER BY er.started_at DESC LIMIT ?`
+    ).all(evalName, sinceTs, limit) : db.prepare(`${baseSql} ORDER BY er.started_at DESC LIMIT ?`).all(evalName, limit);
+    return rows.map((row) => ({
+      runId: row.run_id,
+      timestamp: row.run_timestamp,
+      report: JSON.parse(row.report_json),
+      totalEvalsInRun: row.run_total,
+      filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : void 0
+    }));
+  }
+  async listEvalReportsBulk(opts) {
+    const db = this.getDb();
+    if (!db) return {};
+    const perEval = opts?.perEval ?? 20;
+    const sinceTs = opts?.since;
+    const sinceClause = sinceTs !== void 0 ? "WHERE er.started_at >= ?" : "";
+    const sql = `
+      WITH ranked AS (
+        SELECT er.*, ROW_NUMBER() OVER (PARTITION BY name ORDER BY started_at DESC) AS rn
+        FROM eval_reports er
+        ${sinceClause}
+      )
+      SELECT ranked.run_id, ranked.name, ranked.started_at, ranked.pass,
+             ranked.duration, ranked.type, ranked.error, ranked.report_json,
+             r.timestamp   AS run_timestamp,
+             r.total       AS run_total,
+             r.filter_json AS run_filter_json
+      FROM ranked
+      JOIN eval_runs r ON r.id = ranked.run_id
+      WHERE ranked.rn <= ?
+      ORDER BY ranked.name, ranked.started_at DESC
+    `;
+    const rows = sinceTs !== void 0 ? db.prepare(sql).all(sinceTs, perEval) : db.prepare(sql).all(perEval);
+    const out = {};
+    for (const row of rows) {
+      const entry = {
+        runId: row.run_id,
+        timestamp: row.run_timestamp,
+        report: JSON.parse(row.report_json),
+        totalEvalsInRun: row.run_total,
+        filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : void 0
+      };
+      const arr = out[row.name];
+      if (arr) arr.push(entry);
+      else out[row.name] = [entry];
+    }
+    return out;
+  }
+  // ── Live run tracking ──────────────────────────────────────────────
+  async *watchRun(_signal, _options) {
+  }
+  // ── Runner state ───────────────────────────────────────────────────
+  async getRunnerState() {
+    const runId = this._getActiveRunId();
+    return { running: runId !== null, runId };
+  }
+  // ── Extra (not on EvalStore interface) ──────────────────────────────
+  async listRunResults(limit = 50, sinceTs) {
+    const db = this.getDb();
+    if (!db) return [];
+    let runRows;
+    if (sinceTs !== void 0) {
+      runRows = db.prepare(
+        "SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?"
+      ).all(sinceTs, limit);
+    } else {
+      runRows = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
+    }
+    if (runRows.length === 0) return [];
+    const ids = runRows.map((r) => r.id);
+    const placeholders = ids.map(() => "?").join(",");
+    const reportRows = db.prepare(`SELECT * FROM eval_reports WHERE run_id IN (${placeholders}) ORDER BY seq`).all(...ids);
+    const byRun = /* @__PURE__ */ new Map();
+    for (const row of reportRows) {
+      const arr = byRun.get(row.run_id);
+      if (arr) arr.push(row);
+      else byRun.set(row.run_id, [row]);
+    }
+    return runRows.map((row) => this.assembleRun(row, byRun.get(row.id) ?? []));
+  }
+  // ── Lifecycle ──────────────────────────────────────────────────────
+  close() {
+    try {
+      this.db?.close();
+    } catch {
+    }
+    this.db = null;
+    this.insertRunStmt = null;
+    this.insertReportStmt = null;
+  }
+  getDbPath() {
+    return this.dbPath;
+  }
+  // ── SQLite internals ───────────────────────────────────────────────
+  getDb() {
+    if (this.db) return this.db;
+    mkdirSync(this.evalsDir, { recursive: true });
+    try {
+      this.db = new Database(this.dbPath);
+      this.db.run("PRAGMA journal_mode=WAL");
+      this.db.run("PRAGMA synchronous=NORMAL");
+      this.db.run("PRAGMA busy_timeout=5000");
+      this.db.run("PRAGMA foreign_keys=ON");
+      this.db.run(SCHEMA);
+      this.setMeta("schema_version", SCHEMA_VERSION);
+      this.insertRunStmt = this.db.prepare(`
+        INSERT OR REPLACE INTO eval_runs (
+          id, timestamp, started_at, passed, failed, total, duration,
+          bot_duration, eval_duration, aborted, filter_json, eval_names
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+      this.insertReportStmt = this.db.prepare(`
+        INSERT OR REPLACE INTO eval_reports (
+          run_id, seq, name, started_at, pass, duration, type, error, report_json
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+      this.importLegacyJsonRunsIfNeeded();
+      return this.db;
+    } catch (err) {
+      console.warn("[local-eval-store] Failed to initialize database:", err);
+      try {
+        this.db?.close();
+      } catch {
+      }
+      try {
+        if (existsSync2(this.dbPath)) {
+          renameSync(this.dbPath, `${this.dbPath}.corrupt-${Date.now()}`);
+        }
+      } catch {
+      }
+      this.db = null;
+      this.insertRunStmt = null;
+      this.insertReportStmt = null;
+      return null;
+    }
+  }
+  getMeta(key) {
+    const db = this.db;
+    if (!db) return null;
+    const row = db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(key);
+    return row?.value ?? null;
+  }
+  setMeta(key, value) {
+    const db = this.db;
+    if (!db) return;
+    db.prepare("INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)").run(key, value);
+  }
+  importLegacyJsonRunsIfNeeded() {
+    const db = this.db;
+    if (!db) return;
+    if (this.getMeta("imported_legacy") === "1") return;
+    const legacyDir = join(this.evalsDir, "runs");
+    if (!existsSync2(legacyDir)) {
+      this.setMeta("imported_legacy", "1");
+      return;
+    }
+    let files = [];
+    try {
+      files = readdirSync2(legacyDir).filter((f) => f.endsWith(".json"));
+    } catch (err) {
+      console.warn("[local-eval-store] Could not read legacy runs dir:", err);
+      return;
+    }
+    if (files.length === 0) {
+      this.setMeta("imported_legacy", "1");
+      return;
+    }
+    let imported = 0;
+    let skipped = 0;
+    db.run("BEGIN");
+    try {
+      for (const file of files) {
+        db.run("SAVEPOINT import_file");
+        try {
+          const raw = readFileSync(join(legacyDir, file), "utf-8");
+          const report = JSON.parse(raw);
+          this.insertRunReport(report);
+          db.run("RELEASE import_file");
+          imported++;
+        } catch (err) {
+          db.run("ROLLBACK TO import_file");
+          db.run("RELEASE import_file");
+          skipped++;
+          console.warn(`[local-eval-store] Skipping unreadable legacy run ${file}:`, err);
+        }
+      }
+      this.setMeta("imported_legacy", "1");
+      db.run("COMMIT");
+    } catch (err) {
+      db.run("ROLLBACK");
+      console.warn("[local-eval-store] Legacy import failed, will retry on next start:", err);
+      return;
+    }
+    try {
+      renameSync(legacyDir, join(this.evalsDir, "runs.json-migrated"));
+    } catch (err) {
+      console.warn("[local-eval-store] Could not rename legacy runs dir:", err);
+    }
+    console.log(
+      `[local-eval-store] Imported ${imported} legacy eval run${imported === 1 ? "" : "s"}` + (skipped > 0 ? ` (${skipped} skipped)` : "")
+    );
+  }
+  insertRunReport(report) {
+    if (!this.insertRunStmt || !this.insertReportStmt) return;
+    const startedAt = Date.parse(report.timestamp);
+    const { botDuration, evalDuration } = aggregateTurnTimings(report);
+    this.insertRunStmt.run(
+      report.id,
+      report.timestamp,
+      Number.isNaN(startedAt) ? 0 : startedAt,
+      report.passed,
+      report.failed,
+      report.total,
+      report.duration,
+      botDuration,
+      evalDuration,
+      report.aborted ? 1 : 0,
+      report.filter ? JSON.stringify(report.filter) : null,
+      JSON.stringify(report.evals.map((e) => e.name))
+    );
+    for (let i = 0; i < report.evals.length; i++) {
+      const evalReport = report.evals[i];
+      this.insertReportStmt.run(
+        report.id,
+        i,
+        evalReport.name,
+        Number.isNaN(startedAt) ? 0 : startedAt,
+        evalReport.pass ? 1 : 0,
+        evalReport.duration,
+        evalReport.type ?? null,
+        evalReport.error ?? null,
+        JSON.stringify(evalReport)
+      );
+    }
+  }
+  assembleRun(runRow, reportRows) {
+    const evals = reportRows.map((r) => JSON.parse(r.report_json));
+    const report = {
+      id: runRow.id,
+      timestamp: runRow.timestamp,
+      evals,
+      passed: runRow.passed,
+      failed: runRow.failed,
+      total: runRow.total,
+      duration: runRow.duration
+    };
+    if (runRow.filter_json) {
+      report.filter = JSON.parse(runRow.filter_json);
+    }
+    if (runRow.aborted === 1) {
+      report.aborted = true;
+    }
+    return report;
+  }
+};
+var stores = /* @__PURE__ */ new Map();
+function getLocalEvalStore(agentPath, getActiveRunId) {
+  let store = stores.get(agentPath);
+  if (!store) {
+    store = new LocalEvalStore({ agentPath, getActiveRunId });
+    stores.set(agentPath, store);
+  }
+  return store;
+}
+function closeLocalEvalStores(agentPath) {
+  if (agentPath) {
+    stores.get(agentPath)?.close();
+    stores.delete(agentPath);
+    return;
+  }
+  for (const store of stores.values()) store.close();
+  stores.clear();
+}
+
+// src/stores/vortex-eval-store.ts
+var OUTCOME_TURN_INDEX = -1;
+function vortexEntryToEvalReport(entry) {
+  const outcomeAssertions = entry.results.filter((r) => r.turnIndex === OUTCOME_TURN_INDEX).map((r) => ({
+    assertion: r.graderName,
+    pass: r.passed,
+    expected: r.evidence?.expected ?? "",
+    actual: r.evidence?.actual ?? ""
+  }));
+  const resultsByTurn = /* @__PURE__ */ new Map();
+  for (const r of entry.results) {
+    if (r.turnIndex === OUTCOME_TURN_INDEX) continue;
+    const existing = resultsByTurn.get(r.turnIndex) ?? [];
+    existing.push(r);
+    resultsByTurn.set(r.turnIndex, existing);
+  }
+  const turns = [...resultsByTurn.entries()].sort(([a], [b]) => a - b).map(([turnIndex, results]) => {
+    const assertions = results.map((r) => ({
+      assertion: r.graderName,
+      pass: r.passed,
+      expected: r.evidence?.expected ?? "",
+      actual: r.evidence?.actual ?? ""
+    }));
+    const first = results[0];
+    return {
+      turnIndex,
+      userMessage: first.userMessage,
+      botResponse: first.botResponse,
+      assertions,
+      pass: assertions.every((a) => a.pass),
+      botDuration: first.botDurationMs ?? 0,
+      evalDuration: first.graderDurationMs ?? 0
+    };
+  });
+  return {
+    name: entry.evalName,
+    description: entry.description || void 0,
+    type: entry.evalType,
+    tags: entry.tags,
+    turns,
+    outcomeAssertions,
+    pass: entry.passed ?? false,
+    duration: entry.durationMs ?? 0,
+    error: entry.error ?? void 0
+  };
+}
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i]);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+function turnToResultRows(turn) {
+  return graderResultsToRows(turn.assertions, {
+    turnIndex: turn.turnIndex,
+    userMessage: turn.userMessage,
+    botResponse: turn.botResponse,
+    botDurationMs: turn.botDuration,
+    graderDurationMs: turn.evalDuration
+  });
+}
+function outcomeToResultRows(assertions) {
+  return graderResultsToRows(assertions, {
+    turnIndex: OUTCOME_TURN_INDEX,
+    userMessage: "",
+    botResponse: "",
+    botDurationMs: 0,
+    graderDurationMs: 0
+  });
+}
+function graderResultsToRows(assertions, context) {
+  return assertions.map((assertion) => ({
+    ...context,
+    graderName: assertion.assertion,
+    passed: assertion.pass,
+    evidence: { expected: assertion.expected, actual: assertion.actual }
+  }));
+}
+function runMatchesWatchOptions(run, options) {
+  if (options.runId && run.id !== options.runId) return false;
+  if (options.workflowId && run.metadata?.workflowId !== options.workflowId) return false;
+  return true;
+}
+function vortexRunToReport(run) {
+  const evals = run.entries.map(vortexEntryToEvalReport);
+  return {
+    id: run.id,
+    timestamp: run.createdAt,
+    evals,
+    passed: evals.filter((e) => e.pass).length,
+    failed: evals.filter((e) => !e.pass).length,
+    total: evals.length,
+    duration: evals.reduce((s, e) => s + e.duration, 0)
+  };
+}
+function vortexRunToSummary(run) {
+  const startMs = run.startedAt ? new Date(run.startedAt).getTime() : 0;
+  const endMs = run.completedAt ? new Date(run.completedAt).getTime() : 0;
+  if (run.entries) {
+    const evals = run.entries.map(vortexEntryToEvalReport);
+    return {
+      id: run.id,
+      timestamp: run.createdAt,
+      passed: evals.filter((e) => e.pass).length,
+      failed: evals.filter((e) => !e.pass).length,
+      total: evals.length,
+      duration: evals.reduce((s, e) => s + e.duration, 0),
+      botDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.botDuration, s), 0),
+      evalDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.evalDuration, s), 0),
+      evalNames: evals.map((e) => e.name),
+      aborted: false
+    };
+  }
+  return {
+    id: run.id,
+    timestamp: run.createdAt,
+    passed: 0,
+    failed: 0,
+    total: 0,
+    duration: startMs && endMs ? endMs - startMs : 0,
+    botDuration: 0,
+    evalDuration: 0,
+    evalNames: [],
+    aborted: false
+  };
+}
+var VortexEvalStore = class {
+  url;
+  botId;
+  workspaceId;
+  token;
+  _evalManifestId;
+  _loadEvalDefinitions;
+  constructor(config) {
+    this.url = config.url.replace(/\/$/, "");
+    this.botId = config.botId;
+    if (config.workspaceId) this.workspaceId = config.workspaceId;
+    if (config.token) this.token = config.token;
+    this._evalManifestId = config.evalManifestId;
+    this._loadEvalDefinitions = config.loadEvalDefinitions;
+  }
+  // ── Eval definitions (from Files API manifest) ───────────────────────
+  async listEvals(filter) {
+    const defs = await this.getEvalDefinitions();
+    const filtered = defs.filter((e) => {
+      if (filter?.names && !filter.names.includes(e.name)) return false;
+      if (filter?.type && e.type !== filter.type) return false;
+      if (filter?.tags && !filter.tags.every((tag) => e.tags?.includes(tag))) return false;
+      return true;
+    });
+    return filtered.map((e) => ({
+      name: e.name,
+      description: e.description,
+      tags: e.tags || [],
+      type: e.type || "capability",
+      turnCount: e.conversation.length,
+      hasOutcome: !!e.outcome
+    }));
+  }
+  async getEval(name) {
+    const defs = await this.getEvalDefinitions();
+    return defs.find((e) => e.name === name) ?? null;
+  }
+  // ── Run lifecycle — write ───────────────────────────────────────────
+  async createRun(runType = "scheduled", metadata) {
+    if (!this.workspaceId) {
+      throw new Error("workspaceId is required to create Vortex eval runs");
+    }
+    const triggerType = runType === "manual" ? "manual" : "scheduled";
+    const res = await this.fetch(`/v1/evals/bot/${this.botId}/runs`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        evalManifestId: (typeof this._evalManifestId === "function" ? this._evalManifestId() : this._evalManifestId) ?? "",
+        workspaceId: this.workspaceId,
+        triggerType,
+        ...metadata ? { metadata } : {}
+      })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex create run failed: ${res.status} ${text}`);
+    }
+    const { id } = await res.json();
+    return id;
+  }
+  async addRunResults(runId, evalReport) {
+    const entry = {
+      evalName: evalReport.name,
+      evalType: evalReport.type ?? "capability",
+      description: evalReport.description ?? "",
+      tags: evalReport.tags ?? [],
+      passed: evalReport.pass,
+      durationMs: evalReport.duration,
+      error: evalReport.error,
+      results: [...evalReport.turns.flatMap(turnToResultRows), ...outcomeToResultRows(evalReport.outcomeAssertions)]
+    };
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ entries: [entry] })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex ingest entries failed: ${res.status} ${text}`);
+    }
+  }
+  async completeRun(runId, report) {
+    const hasRunError = report.aborted === true || report.evals.some((e) => e.error !== void 0);
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        entries: [],
+        completed: !hasRunError,
+        failed: hasRunError
+      })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex complete run failed: ${res.status} ${text}`);
+    }
+  }
+  // ── Per-turn incremental lifecycle (Option B) ──
+  //
+  // The eval-runner workflow builds each entry up as the eval runs (rather than
+  // writing it whole at the end), so the dev console can stream turn-by-turn
+  // progress. This is the prod ingestion path and requires Vortex to provide:
+  //   startEntry        POST  /runs/{runId}/entries                    → { entries:[{ id }] }  (passed:null, in-progress)
+  //   appendTurnResults POST  /runs/{runId}/entries/{entryId}/results  (idempotent upsert by turnIndex+graderName)
+  //   finalizeEntry     PATCH /runs/{runId}/entries/{entryId}          (sets the verdict)
+  //   markRunComplete   POST  /runs/{runId}/complete                   (status → completed|failed)
+  /** Create an in-progress entry (metadata only, no results) and return its server id. */
+  async startEntry(runId, meta) {
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        entries: [
+          {
+            evalName: meta.evalName,
+            evalType: meta.evalType ?? "capability",
+            description: meta.description ?? "",
+            tags: meta.tags ?? []
+          }
+        ]
+      })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex start entry failed: ${res.status} ${text}`);
+    }
+    const data = await res.json();
+    const entryId = data.entries?.[0]?.id ?? data.id;
+    if (!entryId) {
+      throw new Error("Vortex start entry returned no entry id");
+    }
+    return entryId;
+  }
+  /** Append one turn's grader results to an existing entry. */
+  async appendTurnResults(runId, entryId, turn) {
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ results: turnToResultRows(turn) })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex append results failed: ${res.status} ${text}`);
+    }
+  }
+  /** Append outcome-level grader results to an existing entry. */
+  async appendOutcomeResults(runId, entryId, assertions) {
+    if (assertions.length === 0) return;
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ results: outcomeToResultRows(assertions) })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex append results failed: ${res.status} ${text}`);
+    }
+  }
+  /** Set the verdict on an entry once its eval finishes. */
+  async finalizeEntry(runId, entryId, verdict) {
+    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}`, {
+      method: "PATCH",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        passed: verdict.passed,
+        ...verdict.durationMs !== void 0 ? { durationMs: verdict.durationMs } : {},
+        ...verdict.error !== void 0 ? { error: verdict.error } : {}
+      })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex finalize entry failed: ${res.status} ${text}`);
+    }
+  }
+  /** Flip the run to its terminal status (replaces completeRun's empty-entries hack). */
+  async markRunComplete(runId, opts = {}) {
+    const res = await this.fetch(`/v1/evals/runs/${runId}/complete`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ ...opts.failed !== void 0 ? { failed: opts.failed } : {} })
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex complete run failed: ${res.status} ${text}`);
+    }
+  }
+  // ── Run results — read ─────────────────────────────────────────────
+  async loadRunResult(runId) {
+    const data = await this.fetchJson(`/v1/evals/runs/${runId}`);
+    return vortexRunToReport(data);
+  }
+  async getLatestRun() {
+    const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, { limit: "1" });
+    if (!data.runs.length) return null;
+    const latest = data.runs[0];
+    return this.loadRunResult(latest.id);
+  }
+  async listRunSummaries(opts) {
+    const runs = await this.fetchRunsWithEntries(opts?.limit ?? 50, opts?.since);
+    return runs.map(vortexRunToSummary);
+  }
+  async listEvalReportsByName(evalName, opts) {
+    const limit = opts?.limit ?? 20;
+    const runs = await this.fetchRunsWithEntries(50, opts?.since);
+    const entries = [];
+    for (const run of runs) {
+      if (entries.length >= limit) break;
+      const entry = run.entries.find((e) => e.evalName === evalName);
+      if (entry) {
+        entries.push({
+          runId: run.id,
+          timestamp: run.createdAt,
+          report: vortexEntryToEvalReport(entry),
+          totalEvalsInRun: run.entries.length
+        });
+      }
+    }
+    return entries;
+  }
+  async listEvalReportsBulk(opts) {
+    const per = opts?.perEval ?? 20;
+    const runs = await this.fetchRunsWithEntries(50, opts?.since);
+    const grouped = {};
+    for (const run of runs) {
+      for (const entry of run.entries) {
+        if (!grouped[entry.evalName]) grouped[entry.evalName] = [];
+        if (grouped[entry.evalName].length < per) {
+          grouped[entry.evalName].push({
+            runId: run.id,
+            timestamp: run.createdAt,
+            report: vortexEntryToEvalReport(entry),
+            totalEvalsInRun: run.entries.length
+          });
+        }
+      }
+    }
+    return grouped;
+  }
+  // ── Live run tracking ──────────────────────────────────────────────
+  async *watchRun(signal, options = {}) {
+    const POLL_INTERVAL = 1500;
+    const POLL_TIMEOUT = 60 * 60 * 1e3;
+    const start = Date.now();
+    const hasExpectedRun = !!options.runId || !!options.workflowId;
+    let baselineRunId = null;
+    if (!hasExpectedRun) {
+      try {
+        const initial = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
+          limit: "1"
+        });
+        baselineRunId = initial.runs[0]?.id ?? null;
+      } catch {
+      }
+    }
+    const evalMeta = await this.loadEvalMeta();
+    const startedEvals = /* @__PURE__ */ new Set();
+    const seenTurns = /* @__PURE__ */ new Set();
+    const startedTurns = /* @__PURE__ */ new Set();
+    const finalizedEvals = /* @__PURE__ */ new Set();
+    while (!signal?.aborted && Date.now() - start < POLL_TIMEOUT) {
+      try {
+        const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
+          limit: hasExpectedRun ? "20" : "1"
+        });
+        let latest = null;
+        let raw = null;
+        for (const candidate of data.runs) {
+          if (options.runId && candidate.id !== options.runId) continue;
+          if (options.workflowId && candidate.metadata?.workflowId !== options.workflowId) {
+            const candidateRaw = await this.fetchJson(`/v1/evals/runs/${candidate.id}`);
+            if (!runMatchesWatchOptions(candidateRaw, options)) continue;
+            raw = candidateRaw;
+          }
+          latest = candidate;
+          break;
+        }
+        if (!latest || !hasExpectedRun && latest.id === baselineRunId) {
+          await new Promise((r) => setTimeout(r, POLL_INTERVAL));
+          continue;
+        }
+        raw ??= await this.fetchJson(`/v1/evals/runs/${latest.id}`);
+        const terminal = raw.status === "completed" || raw.status === "failed";
+        for (let i = 0; i < raw.entries.length; i++) {
+          const entry = raw.entries[i];
+          const evalReport = vortexEntryToEvalReport(entry);
+          const meta = evalMeta.get(entry.evalName);
+          const totalTurns = meta?.totalTurns ?? evalReport.turns.length;
+          if (!startedEvals.has(entry.evalName)) {
+            startedEvals.add(entry.evalName);
+            yield { type: "eval_start", evalName: entry.evalName, index: i, totalTurns };
+          }
+          for (const turn of evalReport.turns) {
+            const turnKey = `${entry.evalName}::${turn.turnIndex}`;
+            if (seenTurns.has(turnKey)) continue;
+            seenTurns.add(turnKey);
+            yield {
+              type: "turn_complete",
+              evalName: entry.evalName,
+              evalIndex: i,
+              turnIndex: turn.turnIndex,
+              totalTurns,
+              turnReport: turn
+            };
+          }
+          if (entry.passed != null && !finalizedEvals.has(entry.evalName)) {
+            finalizedEvals.add(entry.evalName);
+            yield { type: "eval_complete", evalName: entry.evalName, index: i, report: evalReport };
+          } else if (entry.passed == null && !terminal) {
+            const nextTurnIndex = evalReport.turns.length;
+            const startKey = `${entry.evalName}::${nextTurnIndex}`;
+            if (nextTurnIndex < totalTurns && !startedTurns.has(startKey)) {
+              startedTurns.add(startKey);
+              yield {
+                type: "turn_start",
+                evalName: entry.evalName,
+                evalIndex: i,
+                turnIndex: nextTurnIndex,
+                totalTurns,
+                userMessage: meta?.userMessages[nextTurnIndex] ?? ""
+              };
+            }
+          }
+        }
+        if (terminal) {
+          const finalReport = vortexRunToReport(raw);
+          for (let i = 0; i < finalReport.evals.length; i++) {
+            const evalReport = finalReport.evals[i];
+            if (finalizedEvals.has(evalReport.name)) continue;
+            finalizedEvals.add(evalReport.name);
+            yield { type: "eval_complete", evalName: evalReport.name, index: i, report: evalReport };
+          }
+          yield {
+            type: "suite_complete",
+            report: finalReport,
+            error: raw.status === "failed" ? "Eval run failed" : void 0
+          };
+          return;
+        }
+      } catch {
+      }
+      await new Promise((r) => setTimeout(r, POLL_INTERVAL));
+    }
+    if (!signal?.aborted) {
+      yield {
+        type: "suite_complete",
+        report: {
+          id: "timeout",
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          evals: [],
+          passed: 0,
+          failed: 0,
+          total: 0,
+          duration: 0
+        },
+        error: "Eval run timed out"
+      };
+    }
+  }
+  // ── Runner state ───────────────────────────────────────────────────
+  async getRunnerState() {
+    try {
+      const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
+        status: "running",
+        limit: "1"
+      });
+      const active = data.runs[0] ?? null;
+      return { running: !!active, runId: active?.id ?? null };
+    } catch {
+      return { running: false, runId: null };
+    }
+  }
+  // ── Internal helpers ───────────────────────────────────────────────
+  async getEvalDefinitions() {
+    if (!this._loadEvalDefinitions) return [];
+    return this._loadEvalDefinitions();
+  }
+  /**
+   * evalName → { totalTurns, userMessages } from the eval definitions, so
+   * watchRun can report `totalTurns` and synthesize `turn_start` events (with
+   * the turn's user message) for the in-flight turn — matching the dev SSE
+   * experience where the pending turn is visible before its results land.
+   */
+  async loadEvalMeta() {
+    try {
+      const defs = await this.getEvalDefinitions();
+      return new Map(
+        defs.map((d) => [
+          d.name,
+          { totalTurns: d.conversation.length, userMessages: d.conversation.map((t) => t.user ?? "") }
+        ])
+      );
+    } catch {
+      return /* @__PURE__ */ new Map();
+    }
+  }
+  async fetchRunsWithEntries(limit, since) {
+    const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
+      limit: String(limit)
+    });
+    let runs = data.runs;
+    if (since) {
+      runs = runs.filter((r) => new Date(r.createdAt).getTime() >= since);
+    }
+    return mapWithConcurrency(runs, 8, (r) => this.fetchJson(`/v1/evals/runs/${r.id}`));
+  }
+  async fetch(path, init) {
+    const target = new URL(`${this.url}${path}`);
+    const headers = new Headers(init?.headers);
+    if (this.token) headers.set("Authorization", `Bearer ${this.token}`);
+    return globalThis.fetch(target, { ...init, headers });
+  }
+  async fetchJson(path, params) {
+    const target = new URL(`${this.url}${path}`);
+    if (params) {
+      for (const [k, v] of Object.entries(params)) target.searchParams.set(k, v);
+    }
+    const headers = new Headers();
+    if (this.token) headers.set("Authorization", `Bearer ${this.token}`);
+    const res = await globalThis.fetch(target, { headers });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Vortex ${res.status}: ${text}`);
+    }
+    return res.json();
+  }
+};
+export {
+  LocalEvalStore,
+  VortexEvalStore,
+  closeLocalEvalStores,
+  createDiskEvalLoader,
+  getLocalEvalStore
+};
+//# sourceMappingURL=index.js.map