@polderlabs/bizar 4.8.0 → 5.0.0

package/bizar-dash/src/server/eval-store.mjs

@@ -0,0 +1,226 @@
+/**
+ * src/server/eval-store.mjs
+ *
+ * v5.0.0 — Persistent storage for eval runs and results.
+ *
+ * Storage: ~/.local/share/bizar/eval/
+ *
+ * Index: ~/.local/share/bizar/eval/.index.json
+ *   { "runs": [{ id, startedAt, finishedAt, suitePath, total, passed, failed }] }
+ *
+ * Per-run: ~/.local/share/bizar/eval/<run-id>.json
+ *   Full run object with results array.
+ */
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync, renameSync, statSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import { homedir } from 'node:os';
+
+// Allow test override
+const STORE_HOME = process.env.BIZAR_STORE_HOME
+  ? process.env.BIZAR_STORE_HOME
+  : join(homedir(), '.local', 'share', 'bizar');
+const EVAL_DIR = join(STORE_HOME, 'eval');
+const INDEX_FILE = join(EVAL_DIR, '.index.json');
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function ensureDir(dir) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+}
+
+function atomicWriteJson(filePath, data) {
+  const tmp = `${filePath}.tmp.${process.pid}`;
+  writeFileSync(tmp, JSON.stringify(data, null, 2) + '\n', 'utf8');
+  renameSync(tmp, filePath);
+}
+
+function safeReadJSON(file, fallback = null) {
+  try {
+    if (!existsSync(file)) return fallback;
+    const text = readFileSync(file, 'utf8');
+    if (!text.trim()) return fallback;
+    return JSON.parse(text);
+  } catch {
+    return fallback;
+  }
+}
+
+function loadIndex() {
+  return safeReadJSON(INDEX_FILE, { runs: [] });
+}
+
+function saveIndex(index) {
+  ensureDir(EVAL_DIR);
+  atomicWriteJson(INDEX_FILE, index);
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Save a completed run to disk.
+ *
+ * @param {object} run
+ * @param {string} run.id
+ * @param {string} run.startedAt
+ * @param {string} run.finishedAt
+ * @param {string} run.suitePath
+ * @param {number} run.total
+ * @param {number} run.passed
+ * @param {number} run.failed
+ * @param {object[]} run.results
+ * @returns {{ ok: boolean, path: string }}
+ */
+export async function saveRun(run) {
+  ensureDir(EVAL_DIR);
+
+  // Write the run file
+  const runPath = join(EVAL_DIR, `${run.id}.json`);
+  atomicWriteJson(runPath, run);
+
+  // Update the index
+  const index = loadIndex();
+  const existingIdx = index.runs.findIndex((r) => r.id === run.id);
+  const indexEntry = {
+    id: run.id,
+    startedAt: run.startedAt,
+    finishedAt: run.finishedAt,
+    suitePath: run.suitePath,
+    total: run.total,
+    passed: run.passed,
+    failed: run.failed,
+  };
+  if (existingIdx >= 0) {
+    index.runs[existingIdx] = indexEntry;
+  } else {
+    index.runs.unshift(indexEntry); // newest first
+  }
+  saveIndex(index);
+
+  return { ok: true, path: runPath };
+}
+
+/**
+ * List recent runs, newest first.
+ *
+ * @param {{ limit?: number }} opts
+ * @returns {Promise<object[]>}
+ */
+export async function listRuns({ limit = 20 } = {}) {
+  const index = loadIndex();
+  return index.runs.slice(0, Math.max(1, limit));
+}
+
+/**
+ * Get a full run by id.
+ *
+ * @param {string} id
+ * @returns {Promise<object | null>}
+ */
+export async function getRun(id) {
+  if (!id) return null;
+  const runPath = join(EVAL_DIR, `${id}.json`);
+  return safeReadJSON(runPath, null);
+}
+
+/**
+ * Compare two runs and categorize fixtures by diff.
+ *
+ * @param {string} id1
+ * @param {string} id2
+ * @returns {Promise<{ improved: object[], regressed: object[], unchanged: object[] }>}
+ */
+export async function compareRuns(id1, id2) {
+  const [run1, run2] = await Promise.all([getRun(id1), getRun(id2)]);
+
+  if (!run1 || !run2) {
+    return { improved: [], regressed: [], unchanged: [], error: 'run not found' };
+  }
+
+  // Build a map of fixtureId -> result for each run
+  const map1 = new Map(run1.results.map((r) => [r.fixtureId, r]));
+  const map2 = new Map(run2.results.map((r) => [r.fixtureId, r]));
+
+  /** @type {object[]} */
+  const improved = [];
+  /** @type {object[]} */
+  const regressed = [];
+  /** @type {object[]} */
+  const unchanged = [];
+
+  // Check all fixtures from run1
+  for (const [fixtureId, result1] of map1) {
+    const result2 = map2.get(fixtureId);
+    if (!result2) {
+      // Fixture disappeared
+      continue;
+    }
+
+    const entry = {
+      fixtureId,
+      run1Ok: result1.ok,
+      run2Ok: result2.ok,
+      run1LatencyMs: result1.latencyMs,
+      run2LatencyMs: result2.latencyMs,
+    };
+
+    if (result1.ok && !result2.ok) {
+      // Was passing, now failing = regressed
+      regressed.push({ ...entry, kind: 'regressed' });
+    } else if (!result1.ok && result2.ok) {
+      // Was failing, now passing = improved
+      improved.push({ ...entry, kind: 'improved' });
+    } else {
+      // Same result
+      unchanged.push({ ...entry, kind: 'unchanged' });
+    }
+  }
+
+  return { improved, regressed, unchanged };
+}
+
+/**
+ * Delete a run by id.
+ *
+ * @param {string} id
+ * @returns {{ ok: boolean }}
+ */
+export async function deleteRun(id) {
+  if (!id) return { ok: false };
+  const runPath = join(EVAL_DIR, `${id}.json`);
+  try {
+    if (existsSync(runPath)) unlinkSync(runPath);
+  } catch { /* ignore */ }
+
+  const index = loadIndex();
+  index.runs = index.runs.filter((r) => r.id !== id);
+  saveIndex(index);
+
+  return { ok: true };
+}
+
+/**
+ * Build a unique run id.
+ *
+ * @returns {string}
+ */
+export function buildRunId() {
+  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+  const rand = Math.random().toString(36).slice(2, 6);
+  return `run_${ts}_${rand}`;
+}
+
+// ── Test reset ────────────────────────────────────────────────────────────────
+
+/**
+ * Wipe all eval data. For test isolation only.
+ */
+export function __resetStoreForTests() {
+  try {
+    const index = loadIndex();
+    for (const r of index.runs) {
+      try { unlinkSync(join(EVAL_DIR, `${r.id}.json`)); } catch { /* ignore */ }
+    }
+    try { unlinkSync(INDEX_FILE); } catch { /* ignore */ }
+  } catch { /* ignore */ }
+}

package/bizar-dash/src/server/eval.mjs

@@ -0,0 +1,347 @@
+/**
+ * src/server/eval.mjs
+ *
+ * v5.0.0 — Eval runner for evaluating AI agent outputs against golden fixtures.
+ *
+ * Supports:
+ *   - contains/notContains substring checks
+ *   - regex pattern matching
+ *   - JSON schema validation
+ *   - maxTokens bounds
+ *   - maxLatencyMs bounds
+ *   - Parallel fixture execution with configurable concurrency
+ */
+
+import { readFileSync, readdirSync } from 'node:fs';
+import { join, extname } from 'node:path';
+
+/**
+ * @typedef {object} Fixture
+ * @property {string} id
+ * @property {string} name
+ * @property {string} description
+ * @property {string} agent
+ * @property {string} prompt
+ * @property {FixtureExpectations} expected
+ * @property {string[]} [tags]
+ */
+
+/**
+ * @typedef {object} FixtureExpectations
+ * @property {string[]} [contains]
+ * @property {string[]} [notContains]
+ * @property {string[]} [regex]
+ * @property {object} [jsonSchema]
+ * @property {number} [maxTokens]
+ * @property {number} [maxLatencyMs]
+ */
+
+/**
+ * @typedef {object} CheckResult
+ * @property {string} kind
+ * @property {boolean} ok
+ * @property {string} [message]
+ */
+
+/**
+ * @typedef {object} EvalResult
+ * @property {string} fixtureId
+ * @property {boolean} ok
+ * @property {CheckResult[]} checks
+ * @property {number} latencyMs
+ * @property {string} content
+ * @property {{ inputTokens: number, outputTokens: number, totalTokens: number }} usage
+ */
+
+/**
+ * @typedef {object} SuiteResult
+ * @property {string} suitePath
+ * @property {number} total
+ * @property {number} passed
+ * @property {number} failed
+ * @property {EvalResult[]} results
+ */
+
+// ── Check implementations ──────────────────────────────────────────────────────
+
+/**
+ * @param {string} content
+ * @param {string[]} needles
+ * @returns {CheckResult[]}
+ */
+function checkContains(content, needles) {
+  return needles.map((needle) => {
+    const ok = content.includes(needle);
+    return {
+      kind: 'contains',
+      ok,
+      message: ok ? undefined : `expected to contain: "${needle}"`,
+    };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {string[]} needles
+ * @returns {CheckResult[]}
+ */
+function checkNotContains(content, needles) {
+  return needles.map((needle) => {
+    const ok = !content.includes(needle);
+    return {
+      kind: 'notContains',
+      ok,
+      message: ok ? undefined : `expected NOT to contain: "${needle}"`,
+    };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {string[]} patterns
+ * @returns {CheckResult[]}
+ */
+function checkRegex(content, patterns) {
+  return patterns.map((pattern) => {
+    let ok = false;
+    let message;
+    try {
+      const re = new RegExp(pattern);
+      ok = re.test(content);
+      message = ok ? undefined : `regex did not match: ${pattern}`;
+    } catch (err) {
+      ok = false;
+      message = `invalid regex: ${pattern} — ${err.message}`;
+    }
+    return { kind: 'regex', ok, message };
+  });
+}
+
+/**
+ * @param {string} content
+ * @param {object} schema
+ * @returns {CheckResult}
+ */
+function checkJsonSchema(content, schema) {
+  if (!schema) return { kind: 'jsonSchema', ok: true };
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return { kind: 'jsonSchema', ok: false, message: 'response is not valid JSON' };
+  }
+  // Basic schema validation — check required fields exist
+  const errors = [];
+  if (schema.type === 'object' && typeof parsed !== 'object' || Array.isArray(parsed)) {
+    errors.push(`expected object, got ${typeof parsed}`);
+  }
+  if (schema.properties) {
+    for (const [key, prop] of Object.entries(schema.properties)) {
+      if (parsed[key] === undefined) {
+        if (!schema.required?.includes(key)) continue;
+        errors.push(`missing required property: ${key}`);
+      } else if (prop.type && typeof parsed[key] !== prop.type) {
+        errors.push(`property "${key}" expected type ${prop.type}, got ${typeof parsed[key]}`);
+      }
+    }
+  }
+  return {
+    kind: 'jsonSchema',
+    ok: errors.length === 0,
+    message: errors.length > 0 ? errors.join('; ') : undefined,
+  };
+}
+
+/**
+ * @param {{ inputTokens: number, outputTokens: number, totalTokens: number }} usage
+ * @param {number} maxTokens
+ * @returns {CheckResult}
+ */
+function checkMaxTokens(usage, maxTokens) {
+  if (maxTokens == null) return { kind: 'maxTokens', ok: true };
+  const ok = usage.totalTokens <= maxTokens;
+  return {
+    kind: 'maxTokens',
+    ok,
+    message: ok ? undefined : `token count ${usage.totalTokens} exceeds max ${maxTokens}`,
+  };
+}
+
+/**
+ * @param {number} latencyMs
+ * @param {number} maxLatencyMs
+ * @returns {CheckResult}
+ */
+function checkMaxLatency(latencyMs, maxLatencyMs) {
+  if (maxLatencyMs == null) return { kind: 'maxLatencyMs', ok: true };
+  const ok = latencyMs <= maxLatencyMs;
+  return {
+    kind: 'maxLatencyMs',
+    ok,
+    message: ok ? undefined : `latency ${latencyMs}ms exceeds max ${maxLatencyMs}ms`,
+  };
+}
+
+// ── Core runner ───────────────────────────────────────────────────────────────
+
+/**
+ * Run a single fixture against an LLM.
+ *
+ * @param {Fixture} fixture
+ * @param {{ llmCall: (prompt: string, opts: {agent: string}) => Promise<{content: string, usage: object}>, timeoutMs?: number }} opts
+ * @returns {Promise<EvalResult>}
+ */
+export async function runFixture(
+  fixture,
+  { llmCall, timeoutMs = 60000 } = {},
+) {
+  if (!llmCall) throw new Error('llmCall is required');
+  const start = Date.now();
+  let content = '';
+  let usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  let timeoutHit = false;
+  let callError = null;
+
+  try {
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+    const result = await Promise.race([
+      llmCall(fixture.prompt, { agent: fixture.agent }),
+      new Promise((_, reject) => ctrl.signal.addEventListener('abort', () => reject(new Error('timeout')))),
+    ]);
+    clearTimeout(timer);
+    content = result.content || '';
+    usage = result.usage || { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  } catch (err) {
+    if (err.message === 'timeout') {
+      timeoutHit = true;
+    } else {
+      callError = err;
+    }
+    content = '';
+    usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+  }
+
+  const latencyMs = Date.now() - start;
+  const checks = [];
+  const expected = fixture.expected || {};
+
+  // Timeout check
+  if (timeoutHit) {
+    checks.push({ kind: 'timeout', ok: false, message: `call exceeded ${timeoutMs}ms` });
+  }
+
+  // Error check (non-timeout error)
+  if (callError) {
+    checks.push({ kind: 'error', ok: false, message: callError.message });
+  }
+
+  // Content checks
+  if (expected.contains?.length) {
+    checks.push(...checkContains(content, expected.contains));
+  }
+  if (expected.notContains?.length) {
+    checks.push(...checkNotContains(content, expected.notContains));
+  }
+  if (expected.regex?.length) {
+    checks.push(...checkRegex(content, expected.regex));
+  }
+  if (expected.jsonSchema) {
+    checks.push(checkJsonSchema(content, expected.jsonSchema));
+  }
+  if (expected.maxTokens != null) {
+    checks.push(checkMaxTokens(usage, expected.maxTokens));
+  }
+  if (expected.maxLatencyMs != null) {
+    checks.push(checkMaxLatency(latencyMs, expected.maxLatencyMs));
+  }
+
+  const ok = checks.every((c) => c.ok);
+
+  return {
+    fixtureId: fixture.id,
+    ok,
+    checks,
+    latencyMs,
+    content,
+    usage,
+  };
+}
+
+/**
+ * Load all JSON fixtures from a directory.
+ *
+ * @param {string} suitePath
+ * @returns {Fixture[]}
+ */
+export function loadFixtures(suitePath) {
+  /** @type {Fixture[]} */
+  const fixtures = [];
+  try {
+    const entries = readdirSync(suitePath, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isFile() && extname(entry.name) === '.json') {
+        const full = join(suitePath, entry.name);
+        try {
+          const raw = readFileSync(full, 'utf8');
+          const parsed = JSON.parse(raw);
+          // Support both single fixture and suite of fixtures
+          if (parsed.id && parsed.prompt) {
+            fixtures.push(parsed);
+          } else if (Array.isArray(parsed.fixtures)) {
+            fixtures.push(...parsed.fixtures);
+          }
+        } catch { /* skip invalid JSON */ }
+      }
+    }
+  } catch { /* dir not found */ }
+  return fixtures;
+}
+
+/**
+ * Run a suite of fixtures.
+ *
+ * @param {string} suitePath
+ * @param {{ llmCall: Function, concurrency?: number, timeoutMs?: number }} opts
+ * @returns {Promise<SuiteResult>}
+ */
+export async function runSuite(
+  suitePath,
+  { llmCall, concurrency = 5, timeoutMs = 60000 } = {},
+) {
+  const fixtures = loadFixtures(suitePath);
+  /** @type {EvalResult[]} */
+  const results = [];
+  let passed = 0;
+  let failed = 0;
+
+  // Run in batches of `concurrency`
+  for (let i = 0; i < fixtures.length; i += concurrency) {
+    const batch = fixtures.slice(i, i + concurrency);
+    const batchResults = await Promise.all(
+      batch.map((fixture) =>
+        runFixture(fixture, { llmCall, timeoutMs }).catch((err) => ({
+          fixtureId: fixture.id,
+          ok: false,
+          checks: [{ kind: 'error', ok: false, message: err.message }],
+          latencyMs: 0,
+          content: '',
+          usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+        }))
+      )
+    );
+    for (const r of batchResults) {
+      results.push(r);
+      if (r.ok) passed++;
+      else failed++;
+    }
+  }
+
+  return {
+    suitePath,
+    total: fixtures.length,
+    passed,
+    failed,
+    results,
+  };
+}

package/bizar-dash/src/server/memory-lightrag.mjs

@@ -1051,6 +1051,115 @@ function statMaybe(path) {
   }
 }
 
+/**
+ * Get a graph representation of the LightRAG index.
+ *
+ * If `root` is given, fetches entities reachable within `depth` hops
+ * from that entity via LightRAG's /graph endpoint. Otherwise returns
+ * all entities up to `limit`.
+ *
+ * Falls back to reading .bizar/memory/lightrag/kv_store_*.json directly
+ * when the LightRAG server is unavailable.
+ *
+ * @param {{ projectRoot: string, root?: string|null, depth?: number, limit?: number }}
+ * @returns {Promise<{nodes: Array<{id:string,label:string,type:string,size:number,group:string}>, edges: Array<{source:string,target:string,type:string,weight:number}>}>}
+ */
+export async function getLightRAGGraph({ projectRoot, root = null, depth = 2, limit = 200 }) {
+  const config = resolveLightRAGConfig(projectRoot);
+
+  // Attempt to query the running server.
+  if (await isRunning(config)) {
+    try {
+      const nodes = [];
+      const edges = [];
+
+      if (root) {
+        // Use LightRAG's /graph endpoint with root entity.
+        const url = `http://${config.host}:${config.port}/graph?root=${encodeURIComponent(root)}&depth=${depth}&limit=${limit}`;
+        const res = await httpGet(url, config.timeoutMs);
+        if (res.status === 200) {
+          let parsed;
+          try { parsed = JSON.parse(res.body); } catch { parsed = {}; }
+          // LightRAG /graph shape varies; normalise to our contract.
+          // LightRAG returns { entities: [...], relations: [...] } or similar.
+          const ents = parsed.entities || parsed.nodes || [];
+          const rels = parsed.relations || parsed.edges || [];
+          for (const e of ents.slice(0, limit)) {
+            const id = String(e.id || e.name || JSON.stringify(e));
+            nodes.push({
+              id,
+              label: String(e.label || e.name || id).slice(0, 60),
+              type: String(e.type || 'entity'),
+              size: Number(e.size || 1),
+              group: String(e.group || e.type || 'default'),
+            });
+          }
+          for (const r of rels) {
+            edges.push({
+              source: String(r.source || r.from || r.src || ''),
+              target: String(r.target || r.to || r.dst || ''),
+              type: String(r.type || 'related'),
+              weight: Number(r.weight || 1),
+            });
+          }
+          return { nodes, edges };
+        }
+      } else {
+        // Return all entities from kv_store_full_docs.json.
+        const docsPath = join(config.workingDir, 'kv_store_full_docs.json');
+        if (existsSync(docsPath)) {
+          try {
+            const docs = JSON.parse(readFileSync(docsPath, 'utf8'));
+            const keys = Object.keys(docs || {}).slice(0, limit);
+            // Group by top-level path segment as a proxy for group.
+            for (const k of keys) {
+              const parts = k.replace(/^bizar:\/\//, '').split('/');
+              const group = parts.length > 1 ? parts[0] : 'root';
+              nodes.push({
+                id: k,
+                label: parts[parts.length - 1].replace(/[_-]/g, ' '),
+                type: 'note',
+                size: 1,
+                group,
+              });
+            }
+            return { nodes, edges };
+          } catch (err) {
+            console.warn('[lightrag] getLightRAGGraph fallback read failed:', err?.message);
+          }
+        }
+      }
+    } catch (err) {
+      console.warn('[lightrag] getLightRAGGraph server query failed, falling back:', err?.message);
+    }
+  }
+
+  // Fallback: read kv_store files directly.
+  const nodes = [];
+  const edges = [];
+  try {
+    const docsPath = join(config.workingDir, 'kv_store_full_docs.json');
+    if (existsSync(docsPath)) {
+      const docs = JSON.parse(readFileSync(docsPath, 'utf8'));
+      const keys = Object.keys(docs || {}).slice(0, limit);
+      for (const k of keys) {
+        const parts = k.replace(/^bizar:\/\//, '').split('/');
+        const group = parts.length > 1 ? parts[0] : 'root';
+        nodes.push({
+          id: k,
+          label: parts[parts.length - 1].replace(/[_-]/g, ' '),
+          type: 'note',
+          size: 1,
+          group,
+        });
+      }
+    }
+  } catch (err) {
+    console.warn('[lightrag] getLightRAGGraph fallback read error:', err?.message);
+  }
+  return { nodes, edges };
+}
+
 /**
  * Rebuild the graph from scratch. Stops the server, wipes the working dir,
  * and re-runs reindexVault. Returns { ok, started, error?, markerPath? }.