@polderlabs/bizar 4.9.0 → 5.0.0

package/bizar-dash/src/server/eval-store.mjs

@@ -0,0 +1,226 @@
+/**
+ * src/server/eval-store.mjs
+ *
+ * v5.0.0 — Persistent storage for eval runs and results.
+ *
+ * Storage: ~/.local/share/bizar/eval/
+ *
+ * Index: ~/.local/share/bizar/eval/.index.json
+ *   { "runs": [{ id, startedAt, finishedAt, suitePath, total, passed, failed }] }
+ *
+ * Per-run: ~/.local/share/bizar/eval/<run-id>.json
+ *   Full run object with results array.
+ */
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync, renameSync, statSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import { homedir } from 'node:os';
+
+// Allow test override
+const STORE_HOME = process.env.BIZAR_STORE_HOME
+  ? process.env.BIZAR_STORE_HOME
+  : join(homedir(), '.local', 'share', 'bizar');
+const EVAL_DIR = join(STORE_HOME, 'eval');
+const INDEX_FILE = join(EVAL_DIR, '.index.json');
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function ensureDir(dir) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+}
+
+function atomicWriteJson(filePath, data) {
+  const tmp = `${filePath}.tmp.${process.pid}`;
+  writeFileSync(tmp, JSON.stringify(data, null, 2) + '\n', 'utf8');
+  renameSync(tmp, filePath);
+}
+
+function safeReadJSON(file, fallback = null) {
+  try {
+    if (!existsSync(file)) return fallback;
+    const text = readFileSync(file, 'utf8');
+    if (!text.trim()) return fallback;
+    return JSON.parse(text);
+  } catch {
+    return fallback;
+  }
+}
+
+function loadIndex() {
+  return safeReadJSON(INDEX_FILE, { runs: [] });
+}
+
+function saveIndex(index) {
+  ensureDir(EVAL_DIR);
+  atomicWriteJson(INDEX_FILE, index);
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Save a completed run to disk.
+ *
+ * @param {object} run
+ * @param {string} run.id
+ * @param {string} run.startedAt
+ * @param {string} run.finishedAt
+ * @param {string} run.suitePath
+ * @param {number} run.total
+ * @param {number} run.passed
+ * @param {number} run.failed
+ * @param {object[]} run.results
+ * @returns {{ ok: boolean, path: string }}
+ */
+export async function saveRun(run) {
+  ensureDir(EVAL_DIR);
+
+  // Write the run file
+  const runPath = join(EVAL_DIR, `${run.id}.json`);
+  atomicWriteJson(runPath, run);
+
+  // Update the index
+  const index = loadIndex();
+  const existingIdx = index.runs.findIndex((r) => r.id === run.id);
+  const indexEntry = {
+    id: run.id,
+    startedAt: run.startedAt,
+    finishedAt: run.finishedAt,
+    suitePath: run.suitePath,
+    total: run.total,
+    passed: run.passed,
+    failed: run.failed,
+  };
+  if (existingIdx >= 0) {
+    index.runs[existingIdx] = indexEntry;
+  } else {
+    index.runs.unshift(indexEntry); // newest first
+  }
+  saveIndex(index);
+
+  return { ok: true, path: runPath };
+}
+
+/**
+ * List recent runs, newest first.
+ *
+ * @param {{ limit?: number }} opts
+ * @returns {Promise<object[]>}
+ */
+export async function listRuns({ limit = 20 } = {}) {
+  const index = loadIndex();
+  return index.runs.slice(0, Math.max(1, limit));
+}
+
+/**
+ * Get a full run by id.
+ *
+ * @param {string} id
+ * @returns {Promise<object | null>}
+ */
+export async function getRun(id) {
+  if (!id) return null;
+  const runPath = join(EVAL_DIR, `${id}.json`);
+  return safeReadJSON(runPath, null);
+}
+
+/**
+ * Compare two runs and categorize fixtures by diff.
+ *
+ * @param {string} id1
+ * @param {string} id2
+ * @returns {Promise<{ improved: object[], regressed: object[], unchanged: object[] }>}
+ */
+export async function compareRuns(id1, id2) {
+  const [run1, run2] = await Promise.all([getRun(id1), getRun(id2)]);
+
+  if (!run1 || !run2) {
+    return { improved: [], regressed: [], unchanged: [], error: 'run not found' };
+  }
+
+  // Build a map of fixtureId -> result for each run
+  const map1 = new Map(run1.results.map((r) => [r.fixtureId, r]));
+  const map2 = new Map(run2.results.map((r) => [r.fixtureId, r]));
+
+  /** @type {object[]} */
+  const improved = [];
+  /** @type {object[]} */
+  const regressed = [];
+  /** @type {object[]} */
+  const unchanged = [];
+
+  // Check all fixtures from run1
+  for (const [fixtureId, result1] of map1) {
+    const result2 = map2.get(fixtureId);
+    if (!result2) {
+      // Fixture disappeared
+      continue;
+    }
+
+    const entry = {
+      fixtureId,
+      run1Ok: result1.ok,
+      run2Ok: result2.ok,
+      run1LatencyMs: result1.latencyMs,
+      run2LatencyMs: result2.latencyMs,
+    };
+
+    if (result1.ok && !result2.ok) {
+      // Was passing, now failing = regressed
+      regressed.push({ ...entry, kind: 'regressed' });
+    } else if (!result1.ok && result2.ok) {
+      // Was failing, now passing = improved
+      improved.push({ ...entry, kind: 'improved' });
+    } else {
+      // Same result
+      unchanged.push({ ...entry, kind: 'unchanged' });
+    }
+  }
+
+  return { improved, regressed, unchanged };
+}
+
+/**
+ * Delete a run by id.
+ *
+ * @param {string} id
+ * @returns {{ ok: boolean }}
+ */
+export async function deleteRun(id) {
+  if (!id) return { ok: false };
+  const runPath = join(EVAL_DIR, `${id}.json`);
+  try {
+    if (existsSync(runPath)) unlinkSync(runPath);
+  } catch { /* ignore */ }
+
+  const index = loadIndex();
+  index.runs = index.runs.filter((r) => r.id !== id);
+  saveIndex(index);
+
+  return { ok: true };
+}
+
+/**
+ * Build a unique run id.
+ *
+ * @returns {string}
+ */
+export function buildRunId() {
+  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+  const rand = Math.random().toString(36).slice(2, 6);
+  return `run_${ts}_${rand}`;
+}
+
+// ── Test reset ────────────────────────────────────────────────────────────────
+
+/**
+ * Wipe all eval data. For test isolation only.
+ */
+export function __resetStoreForTests() {
+  try {
+    const index = loadIndex();
+    for (const r of index.runs) {
+      try { unlinkSync(join(EVAL_DIR, `${r.id}.json`)); } catch { /* ignore */ }
+    }
+    try { unlinkSync(INDEX_FILE); } catch { /* ignore */ }
+  } catch { /* ignore */ }
+}

package/bizar-dash/src/server/eval.mjs

@@ -0,0 +1,347 @@
+/**
+ * src/server/eval.mjs
+ *
+ * v5.0.0 — Eval runner for evaluating AI agent outputs against golden fixtures.
+ *
+ * Supports:
+ *   - contains/notContains substring checks
+ *   - regex pattern matching
+ *   - JSON schema validation
+ *   - maxTokens bounds
+ *   - maxLatencyMs bounds
+ *   - Parallel fixture execution with configurable concurrency
+ */
+
+import { readFileSync, readdirSync } from 'node:fs';
+import { join, extname } from 'node:path';
+
+/**
+ * @typedef {object} Fixture
+ * @property {string} id
+ * @property {string} name
+ * @property {string} description
+ * @property {string} agent
+ * @property {string} prompt
+ * @property {FixtureExpectations} expected
+ * @property {string[]} [tags]
+ */
+
+/**
+ * @typedef {object} FixtureExpectations
+ * @property {string[]} [contains]
+ * @property {string[]} [notContains]
+ * @property {string[]} [regex]
+ * @property {object} [jsonSchema]
+ * @property {number} [maxTokens]
+ * @property {number} [maxLatencyMs]
+ */
+
+/**
+ * @typedef {object} CheckResult
+ * @property {string} kind
+ * @property {boolean} ok
+ * @property {string} [message]
+ */
+
+/**
+ * @typedef {object} EvalResult
+ * @property {string} fixtureId
+ * @property {boolean} ok
+ * @property {CheckResult[]} checks
+ * @property {number} latencyMs
+ * @property {string} content
+ * @property {{ inputTokens: number, outputTokens: number, totalTokens: number }} usage
+ */
+
+/**
+ * @typedef {object} SuiteResult
+ * @property {string} suitePath
+ * @property {number} total
+ * @property {number} passed
+ * @property {number} failed
+ * @property {EvalResult[]} results
+ */
+
+// ── Check implementations ──────────────────────────────────────────────────────
+
+/**
+ * @param {string} content
+ * @param {string[]} needles
+ * @returns {CheckResult[]}
+ */
+function checkContains(content, needles) {
+  return needles.map((needle) => {
+    const ok = content.includes(needle);
+    return {
+      kind: 'contains',
+      ok,
+      message: ok ? undefined : `expected to contain: "${needle}"`,
+    };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {string[]} needles
+ * @returns {CheckResult[]}
+ */
+function checkNotContains(content, needles) {
+  return needles.map((needle) => {
+    const ok = !content.includes(needle);
+    return {
+      kind: 'notContains',
+      ok,
+      message: ok ? undefined : `expected NOT to contain: "${needle}"`,
+    };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {string[]} patterns
+ * @returns {CheckResult[]}
+ */
+function checkRegex(content, patterns) {
+  return patterns.map((pattern) => {
+    let ok = false;
+    let message;
+    try {
+      const re = new RegExp(pattern);
+      ok = re.test(content);
+      message = ok ? undefined : `regex did not match: ${pattern}`;
+    } catch (err) {
+      ok = false;
+      message = `invalid regex: ${pattern} — ${err.message}`;
+    }
+    return { kind: 'regex', ok, message };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {object} schema
+ * @returns {CheckResult}
+ */
+function checkJsonSchema(content, schema) {
+  if (!schema) return { kind: 'jsonSchema', ok: true };
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return { kind: 'jsonSchema', ok: false, message: 'response is not valid JSON' };
+  }
+  // Basic schema validation — check required fields exist
+  const errors = [];
+  if (schema.type === 'object' && typeof parsed !== 'object' || Array.isArray(parsed)) {
+    errors.push(`expected object, got ${typeof parsed}`);
+  }
+  if (schema.properties) {
+    for (const [key, prop] of Object.entries(schema.properties)) {
+      if (parsed[key] === undefined) {
+        if (!schema.required?.includes(key)) continue;
+        errors.push(`missing required property: ${key}`);
+      } else if (prop.type && typeof parsed[key] !== prop.type) {
+        errors.push(`property "${key}" expected type ${prop.type}, got ${typeof parsed[key]}`);
+      }
+    }
+  }
+  return {
+    kind: 'jsonSchema',
+    ok: errors.length === 0,
+    message: errors.length > 0 ? errors.join('; ') : undefined,
+  };
+}
+
+/**
+ * @param {{ inputTokens: number, outputTokens: number, totalTokens: number }} usage
+ * @param {number} maxTokens
+ * @returns {CheckResult}
+ */
+function checkMaxTokens(usage, maxTokens) {
+  if (maxTokens == null) return { kind: 'maxTokens', ok: true };
+  const ok = usage.totalTokens <= maxTokens;
+  return {
+    kind: 'maxTokens',
+    ok,
+    message: ok ? undefined : `token count ${usage.totalTokens} exceeds max ${maxTokens}`,
+  };
+}
+
+/**
+ * @param {number} latencyMs
+ * @param {number} maxLatencyMs
+ * @returns {CheckResult}
+ */
+function checkMaxLatency(latencyMs, maxLatencyMs) {
+  if (maxLatencyMs == null) return { kind: 'maxLatencyMs', ok: true };
+  const ok = latencyMs <= maxLatencyMs;
+  return {
+    kind: 'maxLatencyMs',
+    ok,
+    message: ok ? undefined : `latency ${latencyMs}ms exceeds max ${maxLatencyMs}ms`,
+  };
+}
+
+// ── Core runner ───────────────────────────────────────────────────────────────
+
+/**
+ * Run a single fixture against an LLM.
+ *
+ * @param {Fixture} fixture
+ * @param {{ llmCall: (prompt: string, opts: {agent: string}) => Promise<{content: string, usage: object}>, timeoutMs?: number }} opts
+ * @returns {Promise<EvalResult>}
+ */
+export async function runFixture(
+  fixture,
+  { llmCall, timeoutMs = 60000 } = {},
+) {
+  if (!llmCall) throw new Error('llmCall is required');
+  const start = Date.now();
+  let content = '';
+  let usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  let timeoutHit = false;
+  let callError = null;
+
+  try {
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+    const result = await Promise.race([
+      llmCall(fixture.prompt, { agent: fixture.agent }),
+      new Promise((_, reject) => ctrl.signal.addEventListener('abort', () => reject(new Error('timeout')))),
+    ]);
+    clearTimeout(timer);
+    content = result.content || '';
+    usage = result.usage || { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  } catch (err) {
+    if (err.message === 'timeout') {
+      timeoutHit = true;
+    } else {
+      callError = err;
+    }
+    content = '';
+    usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  }
+
+  const latencyMs = Date.now() - start;
+  const checks = [];
+  const expected = fixture.expected || {};
+
+  // Timeout check
+  if (timeoutHit) {
+    checks.push({ kind: 'timeout', ok: false, message: `call exceeded ${timeoutMs}ms` });
+  }
+
+  // Error check (non-timeout error)
+  if (callError) {
+    checks.push({ kind: 'error', ok: false, message: callError.message });
+  }
+
+  // Content checks
+  if (expected.contains?.length) {
+    checks.push(...checkContains(content, expected.contains));
+  }
+  if (expected.notContains?.length) {
+    checks.push(...checkNotContains(content, expected.notContains));
+  }
+  if (expected.regex?.length) {
+    checks.push(...checkRegex(content, expected.regex));
+  }
+  if (expected.jsonSchema) {
+    checks.push(checkJsonSchema(content, expected.jsonSchema));
+  }
+  if (expected.maxTokens != null) {
+    checks.push(checkMaxTokens(usage, expected.maxTokens));
+  }
+  if (expected.maxLatencyMs != null) {
+    checks.push(checkMaxLatency(latencyMs, expected.maxLatencyMs));
+  }
+
+  const ok = checks.every((c) => c.ok);
+
+  return {
+    fixtureId: fixture.id,
+    ok,
+    checks,
+    latencyMs,
+    content,
+    usage,
+  };
+}
+
+/**
+ * Load all JSON fixtures from a directory.
+ *
+ * @param {string} suitePath
+ * @returns {Fixture[]}
+ */
+export function loadFixtures(suitePath) {
+  /** @type {Fixture[]} */
+  const fixtures = [];
+  try {
+    const entries = readdirSync(suitePath, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isFile() && extname(entry.name) === '.json') {
+        const full = join(suitePath, entry.name);
+        try {
+          const raw = readFileSync(full, 'utf8');
+          const parsed = JSON.parse(raw);
+          // Support both single fixture and suite of fixtures
+          if (parsed.id && parsed.prompt) {
+            fixtures.push(parsed);
+          } else if (Array.isArray(parsed.fixtures)) {
+            fixtures.push(...parsed.fixtures);
+          }
+        } catch { /* skip invalid JSON */ }
+      }
+    }
+  } catch { /* dir not found */ }
+  return fixtures;
+}
+
+/**
+ * Run a suite of fixtures.
+ *
+ * @param {string} suitePath
+ * @param {{ llmCall: Function, concurrency?: number, timeoutMs?: number }} opts
+ * @returns {Promise<SuiteResult>}
+ */
+export async function runSuite(
+  suitePath,
+  { llmCall, concurrency = 5, timeoutMs = 60000 } = {},
+) {
+  const fixtures = loadFixtures(suitePath);
+  /** @type {EvalResult[]} */
+  const results = [];
+  let passed = 0;
+  let failed = 0;
+
+  // Run in batches of `concurrency`
+  for (let i = 0; i < fixtures.length; i += concurrency) {
+    const batch = fixtures.slice(i, i + concurrency);
+    const batchResults = await Promise.all(
+      batch.map((fixture) =>
+        runFixture(fixture, { llmCall, timeoutMs }).catch((err) => ({
+          fixtureId: fixture.id,
+          ok: false,
+          checks: [{ kind: 'error', ok: false, message: err.message }],
+          latencyMs: 0,
+          content: '',
+          usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+        }))
+      )
+    );
+    for (const r of batchResults) {
+      results.push(r);
+      if (r.ok) passed++;
+      else failed++;
+    }
+  }
+
+  return {
+    suitePath,
+    total: fixtures.length,
+    passed,
+    failed,
+    results,
+  };
+}

package/bizar-dash/src/server/ocr.mjs

@@ -0,0 +1,55 @@
+/**
+ * src/server/ocr.mjs
+ *
+ * v5.0.0 — Tesseract.js wrapper for OCR text extraction.
+ *
+ * Provides a single `extractText(imageBuffer, options?)` function.
+ * The worker is created lazily and terminated after each call to avoid
+ * memory leaks in long-running server processes.
+ */
+
+import { createWorker } from 'tesseract.js';
+
+let worker = null;
+
+/**
+ * Extract text from an image buffer using Tesseract.js OCR.
+ *
+ * @param {Buffer} imageBuffer - Raw image data (PNG, JPEG, etc.)
+ * @param {{ lang?: string }} [options] - Recognition options.
+ *   `lang` defaults to 'eng' (English).
+ * @returns {Promise<string>} - The extracted text.
+ */
+export async function extractText(imageBuffer, { lang = 'eng' } = {}) {
+  const w = worker || (worker = await createWorker(lang));
+
+  try {
+    const { data } = await w.recognize(imageBuffer);
+    return (data.text || '').trim();
+  } finally {
+    // Terminate and reset so a new worker picks up any config changes
+    // on the next call. This prevents the worker from accumulating state
+    // across requests in long-running servers.
+    try { await w.terminate(); } catch { /* ignore */ }
+    worker = null;
+  }
+}
+
+/**
+ * Check whether Tesseract.js is available and can be initialised.
+ * Useful for health-check endpoints.
+ *
+ * @returns {Promise<{ ok: boolean, message: string }>}
+ */
+export async function checkOcrHealth() {
+  try {
+    const w = await createWorker('eng');
+    await w.terminate();
+    return { ok: true, message: 'Tesseract.js worker initialised OK' };
+  } catch (err) {
+    return {
+      ok: false,
+      message: `Tesseract.js init failed: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+}