@polderlabs/bizar 4.4.13 → 4.5.1

package/bizar-dash/src/server/memory-lightrag.mjs

@@ -42,6 +42,8 @@ import {
   closeSync,
   unlinkSync,
   readdirSync,
+  statSync,
+  appendFileSync,
 } from 'node:fs';
 import { join, dirname, basename } from 'node:path';
 import { homedir } from 'node:os';
@@ -99,8 +101,11 @@ export const LIGHTRAG_DEFAULTS = Object.freeze({
   startupTimeoutMs: 30_000,
   llmBinding: 'ollama',     // ollama | openai | lollms | azure_openai | bedrock | gemini
   embeddingBinding: 'ollama', // ollama | openai | azure_openai | bedrock | jina | gemini | voyageai
-  llmModel: 'minimax/MiniMax-M3',
-  embeddingModel: 'text-embedding-3-small',
+  // v4.6.0 — Default to free opencode Zen models (no API key required).
+  // Per-project overrides via .bizar/memory.json#lightrag.{llmModel,embeddingModel}
+  // or env vars BIZAR_LIGHTRAG_LLM / BIZAR_LIGHTRAG_EMBEDDING.
+  llmModel: 'opencode/gpt-5-nano',
+  embeddingModel: 'opencode/text-embedding-3-small',
   llmBindingHost: null,    // for non-ollama bindings
   embeddingBindingHost: null,
   apiKey: 'env',           // 'env' means read from env var; any other string is written to file
@@ -764,4 +769,269 @@ export async function query(config, question, { mode = 'mix', topK = 10 } = {})
   } catch (err) {
     return { ok: false, mode, error: err.message };
   }
+}
+
+// ── v4.7.0 — Memory tab helpers ─────────────────────────────────────────────
+//
+// Lightweight status / stats / rebuild helpers used by the dedicated Memory
+// view. The full reindexVault path above already produces a marker file with
+// most of the numbers we want; these helpers are thin wrappers around it.
+
+/**
+ * Aggregate stats for the Memory → LightRAG panel.
+ *
+ * Pulls from:
+ *   - the last-reindex.json marker file (writeMarker: true leaves it)
+ *   - the configured working dir (where chunks/index files live)
+ *   - the running server (PID, alive)
+ *
+ * Returns a defensive default when nothing has been indexed yet.
+ *
+ * @param {string} projectRoot
+ * @returns {{
+ *   running: boolean,
+ *   pid: number|null,
+ *   host: string,
+ *   port: number,
+ *   workingDir: string,
+ *   lastReindexAt: string|null,
+ *   lastReindexOk: boolean|null,
+ *   lastReindexInserted: number|null,
+ *   lastReindexFailed: number|null,
+ *   noteCount: number,
+ *   indexedApprox: number,
+ *   queryCountLast24h: number,
+ *   avgResponseMs: number|null,
+ *   error?: string,
+ * }}
+ */
+export async function stats(projectRoot) {
+  const config = resolveLightRAGConfig(projectRoot);
+  const cacheDir = join(projectRoot, '.bizar', 'memory-cache');
+  const markerPath = join(cacheDir, 'last-reindex.json');
+  const running = await isRunning(config);
+  let pid = null;
+  const pidFile = join(config.workingDir, 'lightrag.pid');
+  if (existsSync(pidFile)) {
+    const parsed = parseIntSafe(readFileSync(pidFile, 'utf8'));
+    if (parsed !== null && parsed > 0) {
+      try {
+        process.kill(parsed, 0);
+        pid = parsed;
+      } catch {
+        pid = null;
+      }
+    }
+  }
+
+  let lastReindexAt = null;
+  let lastReindexOk = null;
+  let lastReindexInserted = null;
+  let lastReindexFailed = null;
+  if (existsSync(markerPath)) {
+    try {
+      const m = JSON.parse(readFileSync(markerPath, 'utf8'));
+      lastReindexAt = m.finishedAt || m.attemptedAt || null;
+      lastReindexOk = m.ok === true;
+      lastReindexInserted = typeof m.inserted === 'number' ? m.inserted : null;
+      lastReindexFailed = typeof m.failed === 'number' ? m.failed : null;
+    } catch {
+      /* marker corrupt — ignore */
+    }
+  }
+
+  // Approximate the indexed chunk count from the on-disk KV store. LightRAG
+  // writes its chunks to <workingDir>/kv_store_*.json. We sum the lengths of
+  // the document store as a proxy for "indexed documents"; this is best-effort
+  // and clearly labeled as approximate.
+  let indexedApprox = 0;
+  try {
+    const docStore = join(config.workingDir, 'kv_store_doc_status.json');
+    const fullDocs = join(config.workingDir, 'kv_store_full_docs.json');
+    for (const p of [docStore, fullDocs]) {
+      if (existsSync(p)) {
+        try {
+          const obj = JSON.parse(readFileSync(p, 'utf8'));
+          indexedApprox += Object.keys(obj || {}).length;
+        } catch {
+          /* skip */
+        }
+      }
+    }
+  } catch {
+    /* ignore */
+  }
+
+  // Note count = listNotes from the vault.
+  let noteCount = 0;
+  try {
+    const memPath = join(projectRoot, '.bizar', 'memory.json');
+    if (existsSync(memPath)) {
+      const mem = JSON.parse(readFileSync(memPath, 'utf8'));
+      const projectId = mem.projectId || '';
+      const mode = mem.memoryRepo?.mode || 'local-only';
+      let vaultRoot;
+      if (mode === 'local-only') {
+        vaultRoot = join(projectRoot, '.obsidian');
+      } else {
+        const raw = mem.memoryRepo?.path || '';
+        let expanded;
+        if (!raw) expanded = join(projectRoot, '.bizar', 'memory');
+        else if (raw.startsWith('~')) expanded = join(homedir(), raw.slice(1));
+        else if (raw.startsWith('/')) expanded = raw;
+        else expanded = join(projectRoot, raw);
+        vaultRoot = join(expanded, 'projects', projectId);
+      }
+      if (existsSync(vaultRoot)) {
+        const { readdirSync } = await import('node:fs');
+        function walk(dir) {
+          for (const e of readdirSync(dir, { withFileTypes: true })) {
+            if (e.name.startsWith('.')) continue;
+            const full = join(dir, e.name);
+            if (e.isDirectory()) walk(full);
+            else if (e.name.endsWith('.md')) noteCount++;
+          }
+        }
+        walk(vaultRoot);
+      }
+    }
+  } catch {
+    /* ignore — leave noteCount at 0 */
+  }
+
+  // Query stats: best-effort — LightRAG's /query endpoint doesn't expose
+  // query counts in the open-core build. We track our own running counter
+  // in a sidecar JSON file.
+  const queryLogPath = join(cacheDir, 'lightrag-query-log.jsonl');
+  let queryCountLast24h = 0;
+  let totalResponseMs = 0;
+  let counted = 0;
+  if (existsSync(queryLogPath)) {
+    try {
+      const lines = readFileSync(queryLogPath, 'utf8').split('\n').filter(Boolean);
+      const cutoff = Date.now() - 24 * 60 * 60 * 1000;
+      for (const line of lines) {
+        try {
+          const rec = JSON.parse(line);
+          if (typeof rec.ts === 'number' && rec.ts >= cutoff) {
+            queryCountLast24h++;
+            if (typeof rec.ms === 'number' && Number.isFinite(rec.ms)) {
+              totalResponseMs += rec.ms;
+              counted++;
+            }
+          }
+        } catch {
+          /* skip */
+        }
+      }
+    } catch {
+      /* ignore */
+    }
+  }
+  const avgResponseMs = counted > 0 ? Math.round(totalResponseMs / counted) : null;
+
+  return {
+    running,
+    pid: running ? pid : null,
+    host: config.host,
+    port: config.port,
+    workingDir: config.workingDir,
+    lastReindexAt,
+    lastReindexOk,
+    lastReindexInserted,
+    lastReindexFailed,
+    noteCount,
+    indexedApprox,
+    queryCountLast24h,
+    avgResponseMs,
+  };
+}
+
+function parseIntSafe(s) {
+  const n = parseInt(String(s || '').trim(), 10);
+  return Number.isFinite(n) ? n : null;
+}
+
+/**
+ * Record a query for stats tracking. Called by the /query endpoint so the
+ * Memory tab can show usage. Best-effort; never throws.
+ *
+ * @param {string} projectRoot
+ * @param {number} durationMs
+ */
+export function recordQuery(projectRoot, durationMs) {
+  try {
+    const cacheDir = join(projectRoot, '.bizar', 'memory-cache');
+    mkdirSync(cacheDir, { recursive: true });
+    const path = join(cacheDir, 'lightrag-query-log.jsonl');
+    appendFileSafe(path, JSON.stringify({ ts: Date.now(), ms: Math.max(0, Math.round(durationMs)) }) + '\n');
+    // Truncate if file grows past 5MB — keep last ~50k queries.
+    try {
+      const st = statMaybe(path);
+      if (st && st.size > 5 * 1024 * 1024) {
+        const content = readFileSync(path, 'utf8');
+        const lines = content.split('\n');
+        const keep = lines.slice(-50000).join('\n');
+        writeFileSync(path, keep);
+      }
+    } catch {
+      /* ignore */
+    }
+  } catch {
+    /* never throw */
+  }
+}
+
+function appendFileSafe(path, content) {
+  try {
+    appendFileSync(path, content);
+  } catch {
+    /* fall back to manual write */
+    try {
+      const cur = existsSync(path) ? readFileSync(path, 'utf8') : '';
+      writeFileSync(path, cur + content);
+    } catch {
+      /* swallow — stats are best-effort */
+    }
+  }
+}
+
+function statMaybe(path) {
+  try {
+    return statSync(path);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Rebuild the graph from scratch. Stops the server, wipes the working dir,
+ * and re-runs reindexVault. Returns { ok, started, error?, markerPath? }.
+ *
+ * Idempotent: safe to call when the server is already stopped.
+ *
+ * @param {string} projectRoot
+ * @param {{ logger?: { info?: Function, warn?: Function } }} [opts]
+ */
+export async function rebuildGraph(projectRoot, opts = {}) {
+  const config = resolveLightRAGConfig(projectRoot);
+  // Stop server first.
+  await stopServer(config, opts).catch(() => {});
+  // Wipe the working dir contents (preserve the dir itself).
+  try {
+    const { readdirSync } = await import('node:fs');
+    if (existsSync(config.workingDir)) {
+      for (const e of readdirSync(config.workingDir)) {
+        try {
+          unlinkSync(join(config.workingDir, e));
+        } catch {
+          /* skip */
+        }
+      }
+    }
+  } catch {
+    /* ignore */
+  }
+  // Reindex from scratch.
+  return reindexVault(projectRoot, opts);
 }

package/bizar-dash/src/server/memory-obsidian.mjs

@@ -0,0 +1,230 @@
+/**
+ * src/server/memory-obsidian.mjs
+ *
+ * v4.7.0 — Obsidian-flavoured façade for the Bizar Memory Service.
+ *
+ * The Memory tab treats Obsidian as one of four memory sources (alongside
+ * LightRAG, git, and semantic search). This module gives that source a
+ * single import surface that the dashboard can hit without re-deriving the
+ * vault layout or wikilink semantics from raw `memory-store` calls.
+ *
+ * The Obsidian CRUD itself lives in `memory-store.mjs` — re-exporting from
+ * there keeps the schema/secret/git-side-effects behaviour in one place.
+ * This façade adds:
+ *   - tree()         — recursive folder tree for the Memory tab browser
+ *   - listBacklinks() — wikilink reverse index
+ *   - linkGraph()     — node/edge dump for graph visualisation
+ *   - diffVault()     — quick textual diff of two notes
+ *
+ * No new state, no new files on disk — these are all derived from existing
+ * memory-store primitives.
+ */
+
+import { statSync, readdirSync, existsSync } from 'node:fs';
+import { join, basename as pathBasename, relative } from 'node:path';
+import {
+  listNotes,
+  readNote,
+  writeNote,
+  deleteNote,
+  searchVault,
+  findBacklinks as _findBacklinks,
+  vaultStats,
+  resolveVault,
+} from './memory-store.mjs';
+
+// ── Re-exports (verbatim) ──────────────────────────────────────────────────
+
+export {
+  listNotes,
+  readNote,
+  writeNote,
+  deleteNote,
+  searchVault,
+  vaultStats,
+  resolveVault,
+};
+
+// ── Folder tree ────────────────────────────────────────────────────────────
+
+/**
+ * Build a recursive folder tree of the vault, useful for the Memory tab
+ * folder browser. Each node is either a folder (with children) or a leaf
+ * (with size + mtime).
+ *
+ * @param {string} projectRoot
+ * @param {{ maxDepth?: number }} [opts]
+ * @returns {{
+ *   name: string,
+ *   path: string,
+ *   type: 'folder'|'note',
+ *   size?: number,
+ *   mtime?: number,
+ *   children?: Array<ReturnType<typeof tree>>,
+ * }|null}
+ */
+export function tree(projectRoot, { maxDepth = 6 } = {}) {
+  const info = vaultStats(projectRoot);
+  if (!info.exists || !info.vaultRoot || !existsSync(info.vaultRoot)) return null;
+  return buildNode(info.vaultRoot, info.vaultRoot, maxDepth, 0);
+}
+
+function buildNode(absPath, vaultRoot, maxDepth, depth) {
+  const stat = statMaybe(absPath);
+  const name = pathBasename(absPath) || absPath;
+  const rel = relative(vaultRoot, absPath).split('\\').join('/');
+
+  if (stat && stat.isFile()) {
+    return {
+      name: name.replace(/\.md$/i, ''),
+      path: rel || name,
+      type: 'note',
+      size: stat.size,
+      mtime: stat.mtimeMs,
+    };
+  }
+
+  // Directory
+  const children = [];
+  if (depth < maxDepth) {
+    let entries;
+    try {
+      entries = readdirSync(absPath, { withFileTypes: true });
+    } catch {
+      entries = [];
+    }
+    for (const e of entries) {
+      if (e.name.startsWith('.')) continue;
+      const childPath = join(absPath, e.name);
+      children.push(buildNode(childPath, vaultRoot, maxDepth, depth + 1));
+    }
+    children.sort((a, b) => {
+      if (a.type !== b.type) return a.type === 'folder' ? -1 : 1;
+      return a.name.localeCompare(b.name);
+    });
+  }
+
+  return {
+    name,
+    path: rel,
+    type: 'folder',
+    children,
+  };
+}
+
+function statMaybe(p) {
+  try { return statSync(p); } catch { return null; }
+}
+
+// ── Backlinks ──────────────────────────────────────────────────────────────
+
+/**
+ * List notes that link TO `targetRelPath` via wikilinks.
+ *
+ * @param {string} projectRoot
+ * @param {string} targetRelPath
+ * @returns {Array<{ fromRelPath: string, fromTitle: string, snippet: string, mtime: number }>}
+ */
+export function listBacklinks(projectRoot, targetRelPath) {
+  return _findBacklinks(projectRoot, targetRelPath);
+}
+
+// ── Link graph ─────────────────────────────────────────────────────────────
+
+/**
+ * Build a simple node/edge representation of wikilinks across the vault,
+ * suitable for the Memory tab link-graph visualizer.
+ *
+ * Each node is a note (identified by relPath). Each edge is a wikilink
+ * reference from one note to another. Self-links and dangling links
+ * (wikilinks to non-existent notes) are included with `dangling: true`.
+ *
+ * @param {string} projectRoot
+ * @param {{ limit?: number }} [opts]
+ * @returns {{ nodes: Array<{ id: string, label: string, mtime: number }>, edges: Array<{ from: string, to: string, raw: string, dangling: boolean }> }}
+ */
+export function linkGraph(projectRoot, { limit = 500 } = {}) {
+  const notes = listNotes(projectRoot);
+  const known = new Set();
+  for (const n of notes) known.add(n.relPath.replace(/\.md$/i, ''));
+
+  const nodes = notes.slice(0, limit).map((n) => ({
+    id: n.relPath,
+    label: n.frontmatter?.title || pathBasename(n.relPath).replace(/\.md$/i, ''),
+    mtime: n.mtime,
+  }));
+
+  const edges = [];
+  const WIKILINK_RE = /\[\[([^\]\n|#]+?)(?:#[^\]\n|]+?)?(?:\|[^\]\n]+?)?\]\]/g;
+
+  for (const note of notes) {
+    const body = note.body || '';
+    WIKILINK_RE.lastIndex = 0;
+    let m;
+    while ((m = WIKILINK_RE.exec(body)) !== null) {
+      const raw = (m[1] || '').trim();
+      if (!raw) continue;
+      const target = resolveWikilinkTarget(raw, note.relPath);
+      if (!target) continue;
+      const dangling = !known.has(target);
+      edges.push({ from: note.relPath, to: `${target}.md`, raw, dangling });
+      if (edges.length >= limit * 4) break;
+    }
+    if (edges.length >= limit * 4) break;
+  }
+
+  return { nodes, edges };
+}
+
+/**
+ * Resolve a wikilink target string against Obsidian's rules.
+ * Returns the relPath-without-extension if the target exists, or `null` if
+ * the wikilink is empty / unparseable.
+ */
+function resolveWikilinkTarget(raw, fromRelPath) {
+  if (!raw) return null;
+  const stripped = raw.split('#')[0].split('|')[0].trim();
+  if (!stripped) return null;
+  if (stripped.includes('/')) return stripped.replace(/\.md$/i, '');
+  return stripped.replace(/\.md$/i, '');
+}
+
+// ── Diff (textual) ─────────────────────────────────────────────────────────
+
+/**
+ * Lightweight textual diff between two notes. Returns { from, to, lines }.
+ * `lines` is an array of { kind: 'same'|'add'|'del', text } entries.
+ *
+ * Used by the Memory → Git Sync panel to show the working-tree diff of
+ * a single file. Not a full Myers diff — just a sorted-set diff which is
+ * good enough for "what changed?" UI.
+ *
+ * @param {string} projectRoot
+ * @param {string} fromPath — '' for "no previous version"
+ * @param {string} toPath
+ */
+export function diffVault(projectRoot, fromPath, toPath) {
+  const from = fromPath ? readNote(projectRoot, fromPath) : null;
+  const to = toPath ? readNote(projectRoot, toPath) : null;
+  const fromLines = splitLines(from?.body || '');
+  const toLines = splitLines(to?.body || '');
+
+  const fromSet = new Set(fromLines);
+  const toSet = new Set(toLines);
+
+  const lines = [];
+  for (const t of toLines) {
+    if (!fromSet.has(t)) lines.push({ kind: 'add', text: t });
+    else lines.push({ kind: 'same', text: t });
+  }
+  for (const f of fromLines) {
+    if (!toSet.has(f)) lines.push({ kind: 'del', text: f });
+  }
+
+  return { from: fromPath || null, to: toPath || null, lines };
+}
+
+function splitLines(s) {
+  if (!s) return [];
+  return s.split(/\r?\n/).filter((l) => l.length > 0);
+}