mover-os 4.7.7 → 4.7.8

package/src/dashboard/lib/transcript-parser.js

@@ -0,0 +1,331 @@
+"use strict";
+
+// transcript-parser.js — Mover memory engine, Layer 0 (ingestion).
+// STREAMS Claude Code jsonl session transcripts line-by-line (readline) and distils
+// each session into ONE high-signal digest doc: a capped searchable text sample plus
+// runtime artifacts (commands, files, tools, errors, commits) that exist in NO Daily Note.
+//
+// Why streaming, not readFileSync: real sessions reach >512MB → readFileSync throws
+// ERR_STRING_TOO_LONG (Node's max string length). Streaming handles any size with
+// bounded memory — we never hold the whole file, only a capped digest.
+//
+// Reuses the role/ts/content + tool_use extraction shape from src/hooks/session-stub-extractor.js.
+
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const readline = require("readline");
+const { ngramSets, matchLinks } = require("./memory-text");
+
+const DECISION = /\b(decided|let'?s go with|the plan is|we'?ll |going with|instead of|the move is|we should|chose to|verdict|the answer is|conclusion)\b/i;
+// Matches `git [flags] commit [flags] -m|-am "<subject>"`. Flags between git/commit/-m are
+// allowed (e.g. `git -C /repo commit -m`), bounded by no-quote so it can't span across a
+// quoted arg into an unrelated command. The captured subject is validated by extractCommitSubject.
+const COMMIT_RE = /git\b[^"']*\bcommit\b[^"']*-a?m\s+["']([^"']{0,90})/i;
+
+// A git-commit subject from a Bash command, or null. COMMIT_RE matches the `-m "<subject>"`
+// shape, but on the real corpus a commit whose message is a heredoc / command-substitution
+// (`-m "$(cat <<EOF`) captures the shell syntax, not the subject — noise in the commit list.
+// Reject those, bare escapes, and empties; keep only a plausible one-line subject.
+function extractCommitSubject(command) {
+  const m = String(command || "").match(COMMIT_RE);
+  if (!m) return null;
+  const s = (m[1] || "").trim();
+  if (s.length < 3) return null;                 // empty or too short to be a subject
+  if (/^\$\(/.test(s) || /<</.test(s)) return null; // command substitution / heredoc body
+  if (/^[\\\s]+$/.test(s)) return null;          // only backslashes / whitespace
+  return s;
+}
+
+// Slash-command (workflow) detection for error attribution (C1). The transcript form is
+// authoritative: a real invocation is wrapped as <command-name>/log</command-name>. A bare
+// leading "/cmd" is a weaker fallback, constrained so an absolute home path can't
+// masquerade as a command — the command word must end the token (whitespace or EOL after it).
+const CMDTAG_RE = /<command-name>\s*\/?([a-z][\w-]*)/i;
+const SLASH_RE = /^\s*\/([a-z][\w-]*)(?:\s|$)/i;
+function detectCommand(text) {
+  if (typeof text !== "string" || !text) return null;
+  const m = text.match(CMDTAG_RE) || text.match(SLASH_RE);
+  return m ? "/" + m[1].toLowerCase() : null;
+}
+
+const DEFAULT_PROJECTS_DIR = path.join(os.homedir(), ".claude", "projects");
+
+// ── session enumeration (walk + dedup + subagent handling) ──
+
+function deriveProject(dirName) {
+  const i = dirName.indexOf("-01-Projects-");
+  if (i >= 0) return dirName.slice(i + "-01-Projects-".length).replace(/-/g, " ").trim();
+  if (/Architect-OS$/.test(dirName)) return "(vault root)";
+  return dirName.replace(/^-+/, "").replace(/-/g, " ").trim();
+}
+
+function detectAgent(dirName) {
+  const d = dirName.toLowerCase();
+  if (d.includes("gemini") || d.includes("antigravity")) return "gemini";
+  if (d.includes("codex")) return "codex";
+  if (d.includes("cursor")) return "cursor";
+  return "claude-code";
+}
+
+function walkJsonl(dir, projectDir, acc) {
+  let ents;
+  try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return acc; }
+  for (const e of ents) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) {
+      walkJsonl(p, projectDir, acc);
+    } else if (e.name.endsWith(".jsonl")) {
+      const st = (() => { try { return fs.statSync(p); } catch { return null; } })();
+      if (!st) continue;
+      const isSubagent = /(^|\/)subagents(\/|$)/.test(p) || e.name.startsWith("agent-");
+      acc.push({
+        sessionId: e.name.replace(/\.jsonl$/, ""),
+        file: p,
+        projectDir,
+        project: deriveProject(projectDir),
+        agent: detectAgent(projectDir),
+        size: st.size,
+        mtime: st.mtimeMs,
+        isSubagent,
+      });
+    }
+  }
+  return acc;
+}
+
+// Enumerate every session under ~/.claude/projects, deduped by session UUID.
+// The Windows-mirror dir (C--Users-<user>-…) repeats the SAME UUIDs → we keep the
+// largest copy (most complete) so a session is never indexed twice.
+function enumerateSessions(projectsDir = DEFAULT_PROJECTS_DIR, opts = {}) {
+  const includeSubagents = !!opts.includeSubagents;
+  const top = (() => { try { return fs.readdirSync(projectsDir, { withFileTypes: true }); } catch { return []; } })();
+  const all = [];
+  for (const e of top) {
+    if (!e.isDirectory()) continue;
+    walkJsonl(path.join(projectsDir, e.name), e.name, all);
+  }
+  const byId = new Map();
+  const subagents = [];
+  for (const rec of all) {
+    if (rec.isSubagent) { subagents.push(rec); continue; }
+    const prev = byId.get(rec.sessionId);
+    if (!prev || rec.size > prev.size) byId.set(rec.sessionId, rec);
+  }
+  const sessions = [...byId.values()].sort((a, b) => b.mtime - a.mtime);
+  return includeSubagents ? sessions.concat(subagents) : sessions;
+}
+
+// ── single-session parse (streaming) ──
+
+function clampDuration(firstTs, lastTs) {
+  if (!firstTs || !lastTs) return null;
+  const d = (Date.parse(lastTs) - Date.parse(firstTs)) / 60000;
+  // Guard the cross-machine / resumed-session artifact (the proof saw 155,757 min ≈ 108 days).
+  if (!Number.isFinite(d) || d < 0 || d > 72 * 60) return null;
+  return Math.round(d);
+}
+
+// Pick a readable snippet: prefer a high-signal decision, else the first prose-like sentence in
+// the sample, skipping shell/heredoc/code noise (e.g. "$(cat <<…") that makes recall unreadable.
+function pickSnippet(decisions, sampleText) {
+  const readable = (s) => {
+    const t = String(s || "").trim();
+    if (t.length < 16) return false;
+    if (/^[$<`{[(]/.test(t)) return false;
+    if (/\$\(|<<|```|^\s*git\b/.test(t.slice(0, 28))) return false;
+    const letters = (t.match(/[a-z]/gi) || []).length;
+    return letters / t.length > 0.45;
+  };
+  for (const d of decisions || []) if (readable(d)) return d.trim().replace(/\s+/g, " ").slice(0, 220);
+  for (const s of String(sampleText || "").split(/(?<=[.!?])\s+|\n+/)) {
+    if (readable(s)) return s.trim().replace(/\s+/g, " ").slice(0, 220);
+  }
+  return String(sampleText || "").replace(/\s+/g, " ").trim().slice(0, 220);
+}
+
+// Parse one transcript file → a single digest doc. Resolves (never rejects) so a bad
+// file degrades to {error:true} instead of killing a backfill over thousands of files.
+function parseTranscript(file, opts = {}) {
+  const vocab = opts.vocab || null;
+  const MAX = opts.maxSampleChars || 1_200_000;
+  const ASSISTANT_BUDGET = opts.assistantBudget || 300_000; // assistant prose is low-signal; cap hard.
+
+  return new Promise((resolve) => {
+    const turns = { user: 0, assistant: 0 };
+    const tools = Object.create(null);
+    const files = new Set();
+    const commands = [];
+    const commits = [];
+    const errors = [];
+    const decisions = [];
+    let firstTs = null, lastTs = null;
+    // C1 — workflow attribution. currentCommand = the most recent slash-command seen; on each
+    // error we snapshot it into lastCommand so a digest carries WHICH workflow was running when
+    // things broke. Self-improvement (GEPA/§5.5) reads this to name the workflow to fix.
+    let currentCommand = null, lastCommand = null;
+
+    // sampleParts = the high-signal digest (user turns + decisions + artifacts first,
+    // assistant snippets only while budget remains). chars tracks the running total.
+    const sampleParts = [];
+    let chars = 0;
+    let assistantChars = 0;
+    let truncated = false;
+    const push = (t) => {
+      if (!t) return;
+      if (chars >= MAX) { truncated = true; return; }
+      sampleParts.push(t);
+      chars += t.length;
+    };
+
+    let stream;
+    try { stream = fs.createReadStream(file, "utf8"); } catch { return resolve({ file, error: true }); }
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+    rl.on("line", (line) => {
+      if (!line || !line.trim()) return;
+      let obj; try { obj = JSON.parse(line); } catch { return; }
+      // A line can be VALID JSON yet not an object — `null`, a scalar, or an array (corruption /
+      // partial writes). `null.message` below would throw UNCAUGHT in this readline handler and kill
+      // an entire backfill (rl.on("error") only catches stream errors, not handler throws). Skip it.
+      if (!obj || typeof obj !== "object" || Array.isArray(obj)) return;
+      const msg = obj.message || obj;
+      const role = msg.role || obj.type || "";
+      const ts = obj.timestamp || obj.created_at || msg.timestamp || "";
+      if (ts) { if (!firstTs) firstTs = ts; lastTs = ts; }
+      const content = msg.content;
+
+      if (role === "user") {
+        turns.user++;
+        if (typeof content === "string") {
+          const cmd = detectCommand(content); // scan even <command-name> turns (not pushed as prose)
+          if (cmd) currentCommand = cmd;
+          if (!content.startsWith("<")) push(content);
+        } else if (Array.isArray(content)) {
+          for (const c of content) {
+            if (!c) continue;
+            if (c.type === "text" && c.text) {
+              const cmd = detectCommand(c.text);
+              if (cmd) currentCommand = cmd;
+              if (!c.text.startsWith("<")) push(c.text);
+            }
+            if (c.type === "tool_result" && c.is_error) {
+              if (currentCommand) lastCommand = currentCommand; // attribute this error to its workflow
+              // tool_result content is a string OR the Messages-API array of blocks. Extract the text
+              // block(s) so the error reads as plain text — NOT a `[{"type":"text",...}]` JSON blob
+              // that would make GEPA signatures/samples unreadable (L1).
+              const t = typeof c.content === "string"
+                ? c.content
+                : Array.isArray(c.content)
+                  ? (c.content.filter((x) => x && x.type === "text" && x.text).map((x) => x.text).join(" ") || JSON.stringify(c.content))
+                  : JSON.stringify(c.content || "");
+              const e = t.replace(/\s+/g, " ").slice(0, 100);
+              if (e) { errors.push(e); push(e); }
+            }
+          }
+        }
+      } else if (role === "assistant") {
+        turns.assistant++;
+        if (Array.isArray(content)) {
+          for (const c of content) {
+            if (!c) continue;
+            if (c.type === "text" && c.text) {
+              // decisions are high-signal — always captured; full prose is budget-capped.
+              for (const s of c.text.split(/(?<=[.!?])\s+/)) {
+                if (DECISION.test(s) && s.length < 180 && decisions.length < 40) {
+                  const d = s.trim();
+                  decisions.push(d);
+                  push(d);
+                }
+              }
+              if (assistantChars < ASSISTANT_BUDGET) {
+                const snip = c.text.slice(0, 240);
+                assistantChars += snip.length;
+                push(snip);
+              }
+            }
+            if (c.type === "tool_use" && c.name) {
+              tools[c.name] = (tools[c.name] || 0) + 1;
+              const inp = c.input || {};
+              if (typeof inp.file_path === "string") files.add(inp.file_path);
+              if (c.name === "Bash" && typeof inp.command === "string") {
+                const cmd = inp.command.replace(/\s+/g, " ").trim().slice(0, 100);
+                if (commands.length < 200) commands.push(cmd);
+                const subj = extractCommitSubject(inp.command);
+                if (subj) commits.push(subj);
+              }
+            }
+          }
+        }
+      }
+    });
+
+    rl.on("close", () => {
+      // Fold compact artifact text into the digest so commits/files/errors are searchable.
+      const fileBases = [...files].map((f) => path.basename(f));
+      const artifactText =
+        "tools " + Object.keys(tools).join(" ") + "\n" +
+        "files " + fileBases.slice(0, 400).join(" ") + "\n" +
+        "commits " + commits.join(" | ") + "\n" +
+        (errors.length ? "errors " + errors.slice(0, 30).join(" | ") : "");
+      sampleParts.push(artifactText);
+
+      const sampleText = sampleParts.join("\n");
+      let candidateLinks = [];
+      if (vocab) {
+        try { candidateLinks = matchLinks(ngramSets(sampleText), vocab); } catch {}
+      }
+      resolve({
+        file,
+        sessionId: opts.sessionId || path.basename(file).replace(/\.jsonl$/, ""),
+        project: opts.project || null,
+        agent: opts.agent || null,
+        firstTs, lastTs,
+        durationMin: clampDuration(firstTs, lastTs),
+        turns,
+        sampleText,
+        snippet: pickSnippet(decisions, sampleText),
+        sampleChars: sampleText.length,
+        truncated,
+        artifacts: {
+          tools,
+          files: fileBases.slice(0, 60),
+          fileCount: files.size,
+          commands: dedupe(commands).slice(0, 12),
+          commits: dedupe(commits).slice(0, 12),
+          errors: dedupe(errors).slice(0, 6),
+        },
+        decisions: dedupe(decisions).slice(0, 12),
+        candidateLinks,
+        meta: { lastCommand }, // C1: workflow active at the last error (null if none) — for attribution
+      });
+    });
+
+    rl.on("error", () => resolve({ file, error: true }));
+  });
+}
+
+function dedupe(a) { return [...new Set(a)]; }
+
+module.exports = { enumerateSessions, parseTranscript, deriveProject, extractCommitSubject, DEFAULT_PROJECTS_DIR };
+
+// ── CLI (standalone testing) ──
+if (require.main === module) {
+  (async () => {
+    const arg = process.argv[2];
+    if (arg === "--enumerate") {
+      const dir = process.argv[3] || DEFAULT_PROJECTS_DIR;
+      const s = enumerateSessions(dir);
+      const totalMB = s.reduce((a, x) => a + x.size, 0) / 1e6;
+      console.log(`${s.length} unique sessions, ${totalMB.toFixed(0)}MB (deduped, subagents excluded)`);
+      for (const x of s.slice(0, 15)) {
+        console.log(`  ${(x.size / 1e6).toFixed(0).padStart(4)}MB  ${x.project.slice(0, 28).padEnd(28)}  ${x.sessionId}`);
+      }
+    } else if (arg) {
+      const r = await parseTranscript(arg, {});
+      console.log(JSON.stringify({ ...r, sampleText: r.sampleText ? r.sampleText.slice(0, 400) + "…" : "" }, null, 2));
+    } else {
+      console.log("usage: node transcript-parser.js <file.jsonl> | --enumerate [projectsDir]");
+    }
+  })();
+}

package/src/dashboard/lib/vault-graph-parser.js

@@ -0,0 +1,205 @@
+"use strict";
+
+// F39 — Vault Wiki-Link Graph
+//
+// Walks the entire vault, parses every markdown file for [[wiki-links]],
+// builds a node-edge graph. Distinct from PublicProofGraph (which shows
+// BEHAVIORAL patterns); this is the actual wiki-link topology of the
+// user's Obsidian vault.
+//
+// Performance: skips node_modules, .git, .obsidian, 04_Archives.
+// Caps total nodes at 600 by default to keep payload reasonable. Wraps
+// links to non-existent files as orphan nodes (visualizes broken refs).
+
+const fs = require("fs");
+const path = require("path");
+
+const DEFAULT_LIMIT = 600;
+const SKIP_DIRS = new Set([
+  "node_modules", ".git", ".github", ".obsidian", ".trash",
+  "04_Archives", ".claude", ".gemini", ".cursor", ".windsurf",
+  "dist", "build", ".vite", ".next", ".cache", ".turbo",
+  ".vscode", ".idea", "coverage",
+  // Source / dev directories at any depth — these contain code, internal
+  // research, and documentation that uses literal `[[Wiki Links]]` as
+  // examples. Not vault knowledge; they pollute the graph with phantom
+  // broken refs to placeholders like `[[Project Name]]`.
+  "src", "dev", "ui", "ui-v2", "ui-legacy",
+]);
+
+// File NAME pattern to skip walking entirely. `_TEMPLATE.md` files exist
+// throughout the structure (Entity templates, project scaffolds) and only
+// contain placeholder wiki-links by design.
+const SKIP_FILE_RE = /^_TEMPLATE/i;
+
+// Directories whose files contain documentation/placeholder wiki-links
+// like [[Project Name]], [[Daily - YYYY-MM-DD]], [[Entity Name]]. Those
+// are template variables, not real broken refs — we still WALK these
+// dirs (so the files appear in the file count) but we don't parse their
+// links into the graph, which used to produce dozens of false-positive
+// "broken" entries.
+const TEMPLATE_DIRS_RE = /(?:^|\/)(?:Templates|_Template Project|template|templates)(?:\/|$)/i;
+
+// Files to skip when they appear by name — these are framework / agent
+// scaffolding, not user knowledge.
+const SKIP_FILES = new Set([
+  "SKILL.md", "AGENTS.md", "CLAUDE.md", "GEMINI.md", "CURSOR.md",
+]);
+
+function walkMarkdown(dir, out, depth = 0) {
+  if (depth > 6) return; // cap recursion
+  let entries;
+  try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+  for (const e of entries) {
+    // Skip ALL dotfiles/dotdirs by default (they're config, not vault content)
+    if (e.name.startsWith(".")) continue;
+    if (e.isDirectory()) {
+      if (SKIP_DIRS.has(e.name)) continue;
+      walkMarkdown(path.join(dir, e.name), out, depth + 1);
+    } else if (e.isFile() && e.name.endsWith(".md")) {
+      if (SKIP_FILES.has(e.name)) continue;
+      if (SKIP_FILE_RE.test(e.name)) continue;
+      out.push(path.join(dir, e.name));
+    }
+  }
+}
+
+// Parse [[wiki-links]] from a markdown blob. Handles aliases:
+//   [[File Name]]
+//   [[File Name|Display Text]]
+//   [[Folder/File]]
+// Returns array of target stems (filename without .md extension).
+function parseWikiLinks(text) {
+  const out = [];
+  const re = /\[\[([^\]|#]+?)(?:[#|][^\]]*)?\]\]/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const raw = m[1].trim();
+    if (!raw) continue;
+    // Strip optional path prefix; we match by filename stem
+    const stem = raw.split("/").pop()?.replace(/\.md$/i, "") || "";
+    if (stem) out.push(stem);
+  }
+  return out;
+}
+
+function fileStem(filePath) {
+  return path.basename(filePath, ".md");
+}
+
+function topFolder(vault, filePath) {
+  const rel = path.relative(vault, filePath);
+  const parts = rel.split(path.sep);
+  return parts[0] || "(root)";
+}
+
+function buildVaultGraph(vault, { limit = DEFAULT_LIMIT } = {}) {
+  if (!vault || !fs.existsSync(vault)) {
+    return { available: false, error: "vault path not found" };
+  }
+
+  const files = [];
+  walkMarkdown(vault, files);
+
+  // Build stem → filePath index for resolving links
+  const stemToFile = new Map();
+  for (const f of files) {
+    const stem = fileStem(f);
+    if (!stemToFile.has(stem)) stemToFile.set(stem, f);
+  }
+
+  const nodes = new Map(); // id → { id, label, folder, fileCount?, brokenLink?, kind }
+  const edges = []; // { source, target }
+  const inDeg = new Map(); // for centrality sizing
+  const outDeg = new Map();
+
+  function ensureNode(stem, fromFilePath) {
+    if (nodes.has(stem)) return;
+    const filePath = stemToFile.get(stem);
+    const folder = filePath ? topFolder(vault, filePath) : (fromFilePath ? topFolder(vault, fromFilePath) : "(orphan)");
+    nodes.set(stem, {
+      id: stem,
+      label: stem,
+      folder,
+      brokenLink: !filePath, // true if this stem is referenced but no file exists
+      kind: filePath ? "file" : "orphan",
+    });
+  }
+
+  for (const f of files) {
+    const stem = fileStem(f);
+    ensureNode(stem, f);
+    let text;
+    try { text = fs.readFileSync(f, "utf8"); } catch { continue; }
+    // Skip wiki-link parsing for template files — they contain
+    // documentation placeholders like [[Project Name]] / [[Daily - YYYY-MM-DD]]
+    // that are not real references and would otherwise inflate brokenLinks.
+    const rel = path.relative(vault, f);
+    if (TEMPLATE_DIRS_RE.test(rel)) continue;
+    const links = parseWikiLinks(text);
+    for (const target of links) {
+      if (target === stem) continue; // skip self-links
+      ensureNode(target, f);
+      edges.push({ source: stem, target });
+      outDeg.set(stem, (outDeg.get(stem) || 0) + 1);
+      inDeg.set(target, (inDeg.get(target) || 0) + 1);
+    }
+  }
+
+  // Compute full-vault stats ACROSS ALL NODES, not just the top-N visible.
+  // Prior bug: folderCounts / brokenLinks / orphans were derived from the
+  // capped `ranked` slice → "orphans = 0" because the top-N by definition
+  // have non-zero degree, and folder bars summed to ~200 not 1,800+.
+  const fullFolderCounts = {};
+  let totalOrphans = 0;
+  let totalBroken = 0;
+  for (const n of nodes.values()) {
+    const deg = (inDeg.get(n.id) || 0) + (outDeg.get(n.id) || 0);
+    if (n.brokenLink) {
+      totalBroken++;
+      // Don't count broken-link stems in folder distribution — they
+      // aren't real files, they're dangling references.
+      continue;
+    }
+    if (deg === 0) totalOrphans++;
+    fullFolderCounts[n.folder] = (fullFolderCounts[n.folder] || 0) + 1;
+  }
+
+  // Rank nodes by total degree, keep top N (for any UI that wants the
+  // most-connected slice; the totals above are over ALL nodes).
+  const ranked = Array.from(nodes.values())
+    .map(n => ({ ...n, degree: (inDeg.get(n.id) || 0) + (outDeg.get(n.id) || 0), inDegree: inDeg.get(n.id) || 0, outDegree: outDeg.get(n.id) || 0 }))
+    .sort((a, b) => b.degree - a.degree)
+    .slice(0, limit);
+
+  // Surface a separate slice of ALL real orphans + ALL broken so the UI
+  // can show concrete examples without consumers re-walking the graph.
+  const allOrphans = Array.from(nodes.values())
+    .filter(n => !n.brokenLink && ((inDeg.get(n.id) || 0) + (outDeg.get(n.id) || 0)) === 0)
+    .map(n => ({ ...n, degree: 0, inDegree: 0, outDegree: 0 }));
+  const allBroken = Array.from(nodes.values())
+    .filter(n => n.brokenLink)
+    .map(n => ({ ...n, degree: (inDeg.get(n.id) || 0) + (outDeg.get(n.id) || 0), inDegree: inDeg.get(n.id) || 0, outDegree: outDeg.get(n.id) || 0 }))
+    .sort((a, b) => b.inDegree - a.inDegree);
+
+  const keepIds = new Set(ranked.map(n => n.id));
+  const filteredEdges = edges.filter(e => keepIds.has(e.source) && keepIds.has(e.target));
+
+  return {
+    available: true,
+    nodes: ranked,
+    edges: filteredEdges,
+    totalFiles: files.length,
+    totalNodes: nodes.size,
+    totalEdges: edges.length,
+    visibleNodes: ranked.length,
+    visibleEdges: filteredEdges.length,
+    folderCounts: fullFolderCounts,
+    brokenLinks: totalBroken,
+    orphanCount: totalOrphans,
+    orphans: allOrphans.slice(0, 24),
+    brokenSamples: allBroken.slice(0, 24),
+  };
+}
+
+module.exports = { buildVaultGraph, parseWikiLinks };