mover-os 4.7.6 → 4.7.8

package/src/dashboard/lib/memory-index.js

@@ -0,0 +1,470 @@
+"use strict";
+
+// memory-index.js — Mover memory engine, Layer 1 (the index).
+// Zero-dep, pure-JS inverted index + BM25-lite, SHARDED by month so "index everything"
+// never becomes one >512MB string (Node's max-string wall the proof already hit).
+//
+// Layout under ~/.mover/memory/:
+//   manifest.json            global stats: docFreq (for IDF), totalDocs, avgDocLen,
+//                            indexedSessions (incremental skip), linkFreq (gardener/IDF demo)
+//   segments/curated.json    curated docs (session-log arcs, Library, Engine, Auto_Learnings) — always loaded
+//   segments/raw-YYYY-MM.json raw session digests by month — lazy-loaded newest-first
+//
+// Scoring = BM25-lite × type-boost (curated-first) × trust. IDF buries ubiquitous terms;
+// type-boost lifts the curated tier above raw transcripts.
+
+const fs = require("fs");
+const path = require("path");
+const { tokenize, termFreq, buildVocab } = require("./memory-text");
+const { enumerateSessions, parseTranscript, deriveProject } = require("./transcript-parser");
+const { parseSessionLogs } = require("./session-log-parser");
+const { moverDir, safeRead, safeReadJson, engineDir } = require("./paths");
+
+// MOVER_MEMORY_DIR override exists so tests can build a hermetic index in a temp dir.
+const MEM_DIR = process.env.MOVER_MEMORY_DIR || path.join(moverDir(), "memory");
+const SEG_DIR = path.join(MEM_DIR, "segments");
+const MANIFEST = path.join(MEM_DIR, "manifest.json");
+const VERSION = 1;
+
+const K1 = 1.2, B = 0.75;
+// Curated-first: the distilled tiers outrank raw transcripts at equal term match.
+const TYPE_BOOST = { sessionlog: 1.5, autolearning: 1.4, library: 1.3, engine: 1.1, session: 1.0 };
+// A doc TITLED about a query term is strongly relevant — lift it over body-only mentions so
+// common topics (low IDF, e.g. "dashboard") still clear the recall gate when titled for them.
+const TITLE_BOOST = 2.5;
+
+const ENGINE_FILES = [
+  "Identity_Prime.md", "Strategy.md", "Active_Context.md", "Goals.md",
+  "Mover_Dossier.md", "Architect_Dossier.md", "Auto_Learnings.md",
+  "Metrics_Log.md", "Someday_Maybe.md",
+];
+
+// ── helpers ──
+
+function ensureDirs() {
+  try { fs.mkdirSync(SEG_DIR, { recursive: true }); } catch {}
+}
+
+function atomicWrite(file, obj) {
+  const tmp = file + ".tmp";
+  fs.writeFileSync(tmp, JSON.stringify(obj));
+  fs.renameSync(tmp, file);
+}
+
+function monthOf(ts, fallbackMs) {
+  let d = ts ? Date.parse(ts) : NaN;
+  if (!Number.isFinite(d)) d = fallbackMs || Date.now();
+  const dt = new Date(d);
+  return `${dt.getUTCFullYear()}-${String(dt.getUTCMonth() + 1).padStart(2, "0")}`;
+}
+
+function newSeg() { return { docs: Object.create(null), postings: Object.create(null) }; }
+
+// Add one doc {id,type,text,title,...} into a segment, updating global docFreq + length stats.
+function addDoc(d, seg, df, lens) {
+  const toks = tokenize(d.text);
+  if (!toks.length) return;
+  const tf = termFreq(toks);
+  const len = toks.length;
+  lens.total += len; lens.count++;
+  const title = (d.title || "").slice(0, 160);
+  // Normalize ts to an ISO string — library/auto-learnings parsers pass mtime NUMBERS.
+  const ts = d.ts == null ? null : (typeof d.ts === "number" ? new Date(d.ts).toISOString() : String(d.ts));
+  // Strip a leading echo of the title from the snippet (session-log text starts with its title).
+  let body = d.snippet || d.text;
+  if (d.title && body.startsWith(d.title)) body = body.slice(d.title.length);
+  seg.docs[d.id] = {
+    id: d.id, type: d.type, title,
+    project: d.project || null, ts, len,
+    snippet: body.replace(/\s+/g, " ").trim().slice(0, 220),
+    meta: d.meta || {},
+  };
+  for (const term in tf) {
+    (seg.postings[term] || (seg.postings[term] = Object.create(null)))[d.id] = tf[term];
+    df[term] = (df[term] || 0) + 1;
+  }
+}
+
+// Remove ONE doc's contribution from a segment + the global manifest stats (postings, docFreq,
+// totalDocs/Len, linkFreq). Used on re-index AND on the cross-month cleanup (H1). Defensive: clamps
+// docFreq/linkFreq at 0 so a previously-inconsistent index can't drive a term's df negative (which
+// would invert its IDF — M2). Caller persists the mutated segment.
+function removeDocContribution(seg, id, man) {
+  const old = seg.docs[id];
+  if (!old) return;
+  for (const term in seg.postings) {
+    if (seg.postings[term][id] != null) {
+      delete seg.postings[term][id];
+      man.docFreq[term] = Math.max(0, (man.docFreq[term] || 0) - 1);
+      if (man.docFreq[term] <= 0) delete man.docFreq[term];
+      if (!Object.keys(seg.postings[term]).length) delete seg.postings[term];
+    }
+  }
+  man.totalDocs = Math.max(0, (man.totalDocs || 1) - 1);
+  man.totalLen = Math.max(0, (man.totalLen || old.len || 0) - (old.len || 0));
+  for (const note of new Set((old.meta && old.meta.candidateLinks) || [])) {
+    man.linkFreq[note] = Math.max(0, (man.linkFreq[note] || 0) - 1);
+    if (man.linkFreq[note] <= 0) delete man.linkFreq[note];
+  }
+  delete seg.docs[id];
+}
+
+// ── curated docs ──
+
+function curatedDocs(vault) {
+  const docs = [];
+
+  // 1. Session-log arcs (highest signal — /log already distilled them).
+  try {
+    for (const a of parseSessionLogs(vault).docs) {
+      docs.push({ id: a.id, type: "sessionlog", title: a.title, project: a.project, ts: a.date, text: a.text,
+        meta: { date: a.date, time: a.time, commits: a.commits } });
+    }
+  } catch (e) { /* honest-null: missing dailies just means no arcs */ }
+
+  // 2. Library files (name + first paragraph + wikilinks).
+  try {
+    const { indexLibraryV2 } = require("./library-indexer-v2");
+    const lib = indexLibraryV2(vault);
+    const sections = (lib && lib.sections) || {};
+    for (const key of Object.keys(sections)) {
+      const sec = sections[key];
+      // Sections vary: most have items[]; inputs has byType{}; entities has people/orgs/places/decisions[].
+      let items = [];
+      if (Array.isArray(sec.items)) items = sec.items;
+      else if (sec.byType) items = [].concat(...Object.values(sec.byType));
+      else if (sec.people || sec.orgs || sec.places || sec.decisions)
+        items = [].concat(sec.people || [], sec.orgs || [], sec.places || [], sec.decisions || []);
+      for (const it of items) {
+        if (!it || !it.name) continue;
+        docs.push({ id: `lib:${it.path || it.name}`, type: "library", title: it.name, ts: it.mtime || null,
+          text: [it.name, it.firstParagraph || "", (it.wikilinks || []).join(" ")].join("\n"),
+          meta: { path: it.path, section: key } });
+      }
+    }
+  } catch (e) { /* library indexer optional */ }
+
+  // 3. Engine files (whole-file; ubiquitous so low boost, but searchable for "what's my X").
+  for (const name of ENGINE_FILES) {
+    const p = path.join(engineDir(vault), name);
+    const raw = safeRead(p);
+    if (!raw) continue;
+    docs.push({ id: `engine:${name}`, type: "engine", title: name.replace(/\.md$/, ""), ts: null,
+      text: raw.slice(0, 40000), meta: { path: p } });
+  }
+
+  // 4. Auto_Learnings entries (behavioral patterns w/ confidence).
+  try {
+    const { parseAutoLearnings } = require("./auto-learnings-parser");
+    const al = parseAutoLearnings(vault);
+    for (const e of (al.entries || [])) {
+      if (!e || !e.name) continue;
+      docs.push({ id: `al:${e.type || "x"}:${e.name}`, type: "autolearning", title: e.name, ts: e.lastUpdated || null,
+        text: [e.type, e.name, e.body || ""].join("\n"), meta: { confidence: e.confidence, status: e.status } });
+    }
+  } catch (e) { /* optional */ }
+
+  return docs;
+}
+
+// ── full backfill ──
+
+async function buildIndex(vault, opts = {}) {
+  ensureDirs();
+  const log = opts.log || (() => {});
+  const t0 = Date.now();
+  const vocab = buildVocab(vault);
+  log(`vocab: ${vocab.entries.length} note names`);
+
+  const df = Object.create(null);
+  const lens = { total: 0, count: 0 };
+  const linkFreq = Object.create(null);
+  const indexedSessions = Object.create(null);
+  const segs = Object.create(null); // segName -> seg
+  const getSeg = (name) => segs[name] || (segs[name] = newSeg());
+
+  // curated segment
+  const cur = getSeg("curated");
+  const cdocs = curatedDocs(vault);
+  for (const d of cdocs) addDoc(d, cur, df, lens);
+  log(`curated: ${cdocs.length} docs`);
+
+  // raw session segments (streamed, one session at a time → bounded memory)
+  const sessions = enumerateSessions(opts.projectsDir);
+  log(`raw: ${sessions.length} unique sessions to index`);
+  let n = 0;
+  for (const s of sessions) {
+    const r = await parseTranscript(s.file, { vocab, sessionId: s.sessionId, project: s.project, agent: s.agent });
+    n++;
+    if (r.error) { log(`  [${n}/${sessions.length}] SKIP (read error) ${s.sessionId}`); continue; }
+    const month = monthOf(r.firstTs, s.mtime);
+    const seg = getSeg("raw-" + month);
+    const title = `Session · ${s.project} · ${(r.firstTs || "").slice(0, 10) || month}`;
+    addDoc({
+      id: `session:${s.sessionId}`, type: "session", title, project: s.project, ts: r.firstTs,
+      text: r.sampleText,
+      snippet: r.snippet || r.decisions[0] || r.sampleText,
+      meta: {
+        sessionId: s.sessionId, file: s.file, month, agent: s.agent,
+        turns: r.turns, durationMin: r.durationMin, sizeMB: +(s.size / 1e6).toFixed(1),
+        commits: r.artifacts.commits, commands: r.artifacts.commands, errors: r.artifacts.errors,
+        topTools: Object.keys(r.artifacts.tools).slice(0, 6), candidateLinks: r.candidateLinks,
+        lastCommand: (r.meta && r.meta.lastCommand) || null, // C1: workflow active at the last error (GEPA attribution)
+      },
+    }, seg, df, lens);
+    indexedSessions[s.sessionId] = { size: s.size, mtime: s.mtime, month };
+    // linkFreq: how many sessions mention each note (drives link-IDF / the noise fix).
+    for (const note of new Set(r.candidateLinks)) linkFreq[note] = (linkFreq[note] || 0) + 1;
+    if (n % 25 === 0) log(`  [${n}/${sessions.length}] indexed (${Math.round(process.memoryUsage().rss / 1e6)}MB rss)`);
+  }
+
+  // write segments
+  let bytes = 0;
+  for (const name of Object.keys(segs)) {
+    const file = path.join(SEG_DIR, name + ".json");
+    atomicWrite(file, segs[name]);
+    bytes += fs.statSync(file).size;
+  }
+  // Proactive-recall triggers: entities/projects worth surfacing when merely MENTIONED
+  // (signal only — drop ubiquitous notes and sub-4-char names). Plain newline text so the
+  // bash pre-filter can `grep -wFf` it in ~ms.
+  try {
+    const N = sessions.length || 1;
+    const trig = Object.keys(linkFreq)
+      .filter((n) => n.length >= 4 && linkFreq[n] / N <= 0.5)
+      .sort((a, b) => linkFreq[a] - linkFreq[b]);
+    fs.writeFileSync(path.join(MEM_DIR, "recall-triggers.txt"), trig.join("\n") + (trig.length ? "\n" : ""));
+  } catch {}
+  const manifest = {
+    version: VERSION, builtAt: new Date(t0).toISOString(), vault,
+    totalDocs: lens.count, totalLen: lens.total, avgDocLen: lens.count ? lens.total / lens.count : 0,
+    docFreq: df, linkFreq, indexedSessions,
+    segments: Object.keys(segs), trust: {},
+  };
+  atomicWrite(MANIFEST, manifest);
+
+  return {
+    ms: Date.now() - t0, totalDocs: lens.count, segments: Object.keys(segs).length,
+    indexBytes: bytes, curatedDocs: cdocs.length, sessions: sessions.length,
+    uniqueTerms: Object.keys(df).length, uniqueLinks: Object.keys(linkFreq).length,
+  };
+}
+
+// ── query ──
+
+function loadManifest() { return safeReadJson(MANIFEST); }
+function loadSeg(name) { return safeReadJson(path.join(SEG_DIR, name + ".json")); }
+
+function idf(df, N, term) {
+  const n = df[term] || 0;
+  return Math.log(1 + (N - n + 0.5) / (n + 0.5));
+}
+
+// Scope guard (precision-first PUSH): a project-tagged doc is in-scope only for its own project,
+// so another project's session never bleeds into this one. Vault-global curated docs
+// (engine/library/autolearning, project=null) are always in scope. Lenient on sub-path drift so a
+// session started in a project subdir still matches its root. Opt-in via opts.project — the deep
+// pull skill passes none and searches everything.
+// Normalize away tag punctuation ("[Mover OS Bundle", "Mover OS Bundle + Belgium") so a legit
+// current-project arc isn't filtered over a stray bracket — while a real other project still won't match.
+function normProject(p) { return String(p || "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim(); }
+function inScope(docProject, currentProject) {
+  if (!docProject) return true;       // vault-global curated → always in scope
+  if (!currentProject) return true;   // unknown current project → don't filter (fail-open)
+  const d = normProject(docProject), c = normProject(currentProject);
+  // One-directional, word-boundary match (M1): in scope iff the doc IS the current project, or a
+  // sub-scope of it ("Mover OS Bundle + Belgium" → starts with "mover os bundle "). NOT the reverse:
+  // a shorter prefix-named OTHER project ("Mover") must never bleed into "Mover OS Bundle", and the
+  // trailing space stops a same-prefix sibling ("Mover OS Bundle2") from matching either.
+  return d === c || d.startsWith(c + " ");
+}
+
+function query(qStr, opts = {}) {
+  const man = loadManifest();
+  if (!man) return { available: false, hits: [] };
+  const qTerms = [...new Set(tokenize(qStr))];
+  if (!qTerms.length) return { available: true, hits: [], terms: [] };
+
+  const N = man.totalDocs || 1;
+  const avgdl = man.avgDocLen || 1;
+  const maxSegments = opts.maxSegments || 12;
+  const trust = man.trust || {};
+
+  // Always load curated; then raw segments newest-first up to the budget.
+  const rawSegs = (man.segments || []).filter((s) => s.startsWith("raw-")).sort().reverse().slice(0, maxSegments);
+  const segNames = ["curated", ...rawSegs].filter((s) => (man.segments || []).includes(s));
+
+  const scores = Object.create(null);
+  const docRef = Object.create(null);
+  for (const name of segNames) {
+    const seg = loadSeg(name);
+    if (!seg) continue;
+    for (const term of qTerms) {
+      const post = seg.postings[term];
+      if (!post) continue;
+      const w = idf(man.docFreq, N, term);
+      for (const id in post) {
+        const d = seg.docs[id];
+        if (!d) continue;
+        const tf = post[id];
+        const denom = tf + K1 * (1 - B + B * (d.len / avgdl));
+        const s = w * (tf * (K1 + 1)) / denom;
+        scores[id] = (scores[id] || 0) + s;
+        docRef[id] = d;
+      }
+    }
+  }
+
+  const hits = Object.keys(scores).map((id) => {
+    const d = docRef[id];
+    const boost = TYPE_BOOST[d.type] || 1.0;
+    const tr = trust[id] != null ? trust[id] : 1.0;
+    const titleLc = (d.title || "").toLowerCase();
+    const tBoost = qTerms.some((t) => titleLc.includes(t)) ? TITLE_BOOST : 1.0;
+    return { id, type: d.type, title: d.title, project: d.project, ts: d.ts,
+      score: +(scores[id] * boost * tr * tBoost).toFixed(3), snippet: d.snippet, meta: d.meta };
+  });
+  hits.sort((a, b) => b.score - a.score);
+  // Cross-project scope guard — only when a current project is supplied (push path).
+  const scoped = opts.project ? hits.filter((h) => inScope(h.project, opts.project)) : hits;
+  return { available: true, terms: qTerms, total: scoped.length, hits: scoped.slice(0, opts.limit || 10) };
+}
+
+// ── link-IDF report (the "watch the noise disappear" proof) ──
+
+function linkReport(sessionId, opts = {}) {
+  const man = loadManifest();
+  if (!man) return null;
+  const N = Object.keys(man.indexedSessions || {}).length || 1;
+  // find the session doc (search its month segment, else scan raw segs)
+  let doc = null;
+  for (const name of (man.segments || []).filter((s) => s.startsWith("raw-")).sort().reverse()) {
+    const seg = loadSeg(name);
+    if (seg && seg.docs["session:" + sessionId]) { doc = seg.docs["session:" + sessionId]; break; }
+  }
+  if (!doc) return null;
+  const links = (doc.meta.candidateLinks || []).map((note) => {
+    const freq = man.linkFreq[note] || 1;
+    const lidf = Math.log(N / freq);
+    return { note, sessions: freq, idf: +lidf.toFixed(2), score: +lidf.toFixed(2) };
+  });
+  const rawOrder = [...links].sort((a, b) => b.sessions - a.sessions); // "naive" = most-mentioned first
+  const idfOrder = [...links].sort((a, b) => b.idf - a.idf);            // IDF-weighted = rarest first
+  return { sessionId, total: links.length, N, naive: rawOrder.slice(0, 12), idf: idfOrder.slice(0, 12) };
+}
+
+function stats() {
+  const man = loadManifest();
+  if (!man) return { available: false };
+  let bytes = 0;
+  for (const s of (man.segments || [])) {
+    try { bytes += fs.statSync(path.join(SEG_DIR, s + ".json")).size; } catch {}
+  }
+  return {
+    available: true, builtAt: man.builtAt, totalDocs: man.totalDocs,
+    avgDocLen: +man.avgDocLen.toFixed(1), uniqueTerms: Object.keys(man.docFreq || {}).length,
+    sessions: Object.keys(man.indexedSessions || {}).length, segments: man.segments,
+    uniqueLinks: Object.keys(man.linkFreq || {}).length, indexMB: +(bytes / 1e6).toFixed(1),
+  };
+}
+
+// ── incremental: index ONE session (the session-end path; no full rebuild) ──
+// Idempotent: skips a session whose mtime is unchanged. Re-indexes (remove→add) if the
+// file grew. Updates global docFreq/linkFreq/totals in the manifest directly so IDF stays
+// correct without re-reading the whole corpus.
+async function indexSession(file, opts = {}) {
+  const man = loadManifest();
+  if (!man) return { error: "no index — run --backfill first" };
+  man.docFreq = man.docFreq || {}; man.linkFreq = man.linkFreq || {}; man.indexedSessions = man.indexedSessions || {};
+  const vault = opts.vault || man.vault;
+  const sessionId = opts.sessionId || path.basename(file).replace(/\.jsonl$/, "");
+  // Derive the project from the transcript's parent dir (…/projects/<encoded-dir>/<uuid>.jsonl)
+  // when not passed — the SessionEnd hook calls indexSession(file, {}), and without this every
+  // live-indexed session would be project=null, making the cross-project scope guard a no-op.
+  const project = opts.project || deriveProject(path.basename(path.dirname(file))) || null;
+  let mtime = null, size = 0;
+  try { const st = fs.statSync(file); mtime = st.mtimeMs; size = st.size; } catch {}
+  const prev = man.indexedSessions[sessionId];
+  if (prev && mtime && prev.mtime === mtime && !opts.force) return { skipped: true, sessionId };
+
+  const r = await parseTranscript(file, { vocab: buildVocab(vault), sessionId, project });
+  if (r.error) return { error: "parse failed", sessionId };
+
+  const month = monthOf(r.firstTs, mtime);
+  const id = "session:" + sessionId;
+
+  // H1: a session's month can change between indexings (e.g. first indexed off mtime while its
+  // turns are still timestamp-less, then a real earlier/later timestamp appears). The old copy then
+  // lives in the PREVIOUSLY recorded month's segment — remove it from THERE too, else it orphans
+  // (ghost doc) and permanently inflates docFreq/totalDocs/linkFreq, degrading IDF until a full
+  // --backfill. Only fires on a genuine month change; the common same-month re-index skips it.
+  let reindexed = false;
+  const prevMonth = prev && prev.month;
+  if (prevMonth && prevMonth !== month) {
+    const prevSegFile = path.join(SEG_DIR, "raw-" + prevMonth + ".json");
+    const prevSeg = safeReadJson(prevSegFile);
+    if (prevSeg && prevSeg.docs[id]) {
+      removeDocContribution(prevSeg, id, man);
+      atomicWrite(prevSegFile, prevSeg);
+      reindexed = true;
+    }
+  }
+
+  const segFile = path.join(SEG_DIR, "raw-" + month + ".json");
+  const seg = safeReadJson(segFile) || newSeg();
+
+  // remove old contribution (same-month re-index case) — scan only this segment's postings.
+  if (seg.docs[id]) { removeDocContribution(seg, id, man); reindexed = true; }
+
+  // add the fresh doc + fold its terms into the global stats.
+  const tf = termFreq(tokenize(r.sampleText));
+  const len = Object.values(tf).reduce((a, b) => a + b, 0);
+  seg.docs[id] = {
+    id, type: "session", title: `Session · ${r.project || project || "?"} · ${(r.firstTs || "").slice(0, 10) || month}`,
+    project: r.project || project || null, ts: r.firstTs, len,
+    snippet: r.snippet || (r.decisions[0] || r.sampleText).slice(0, 220).replace(/\s+/g, " "),
+    meta: { sessionId, file, month, turns: r.turns, durationMin: r.durationMin,
+      commits: r.artifacts.commits, commands: r.artifacts.commands, errors: r.artifacts.errors,
+      topTools: Object.keys(r.artifacts.tools).slice(0, 6), candidateLinks: r.candidateLinks,
+      lastCommand: (r.meta && r.meta.lastCommand) || null }, // C1: workflow active at the last error (GEPA attribution)
+  };
+  for (const term in tf) {
+    (seg.postings[term] || (seg.postings[term] = Object.create(null)))[id] = tf[term];
+    man.docFreq[term] = (man.docFreq[term] || 0) + 1;
+  }
+  man.totalDocs = (man.totalDocs || 0) + 1;
+  man.totalLen = (man.totalLen || 0) + len;
+  man.avgDocLen = man.totalDocs ? man.totalLen / man.totalDocs : 0;
+  for (const note of new Set(r.candidateLinks)) man.linkFreq[note] = (man.linkFreq[note] || 0) + 1;
+  man.indexedSessions[sessionId] = { size, mtime, month };
+  if (!man.segments.includes("raw-" + month)) man.segments.push("raw-" + month);
+
+  ensureDirs();
+  atomicWrite(segFile, seg);
+  atomicWrite(MANIFEST, man);
+  return { ok: true, sessionId, month, len, totalDocs: man.totalDocs, reindexed };
+}
+
+module.exports = { buildIndex, indexSession, query, linkReport, stats, inScope, MEM_DIR };
+
+// ── CLI ──
+if (require.main === module) {
+  (async () => {
+    const cmd = process.argv[2];
+    const vault = require("./paths").resolveVault();
+    if (cmd === "--backfill") {
+      const r = await buildIndex(vault, { log: (m) => console.log(m) });
+      console.log("\nDONE:", JSON.stringify(r, null, 2));
+    } else if (cmd === "--query") {
+      console.log(JSON.stringify(query(process.argv.slice(3).join(" "), { limit: 8 }), null, 2));
+    } else if (cmd === "--links") {
+      console.log(JSON.stringify(linkReport(process.argv[3]), null, 2));
+    } else if (cmd === "--index") {
+      console.log(JSON.stringify(await indexSession(process.argv[3], { force: process.argv.includes("--force") }), null, 2));
+    } else if (cmd === "--stats") {
+      console.log(JSON.stringify(stats(), null, 2));
+    } else {
+      console.log('usage: node memory-index.js --backfill | --query "..." | --links <sessionId> | --stats');
+    }
+  })();
+}

package/src/dashboard/lib/memory-rerank.js

@@ -0,0 +1,72 @@
+"use strict";
+
+// memory-rerank.js — Mover memory engine, Layer 7 (optional, local semantic rerank).
+//
+// BM25-lite already retrieves good candidates. IF a local Ollama is running, this reranks the
+// top candidates by embedding cosine — catching paraphrase matches keyword search misses.
+// Strictly local (nomic-embed-text on localhost), $0, and GRACEFULLY DEGRADES to keyword
+// when Ollama is absent. It is NEVER an embedding API — nothing leaves the machine. Uses Node's
+// built-in fetch (zero deps).
+
+const DEFAULT_HOST = process.env.OLLAMA_HOST || "http://127.0.0.1:11434";
+const MODEL = process.env.MOVER_EMBED_MODEL || "nomic-embed-text";
+
+function cosine(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  if (!na || !nb) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+// Embed one string via Ollama. Returns the vector, or null on any failure (absent/slow/error).
+async function embed(text, opts = {}) {
+  const host = opts.host || DEFAULT_HOST;
+  const model = opts.model || MODEL;
+  const timeoutMs = opts.timeoutMs || 1500;
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const res = await fetch(host + "/api/embeddings", {
+      method: "POST", signal: ctrl.signal,
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ model, prompt: String(text || "").slice(0, 4000) }),
+    });
+    clearTimeout(t);
+    if (!res.ok) return null;
+    const j = await res.json();
+    return Array.isArray(j.embedding) ? j.embedding : null;
+  } catch { clearTimeout(t); return null; }
+}
+
+// Rerank BM25 hits by semantic similarity IF Ollama is up; otherwise return them untouched.
+// Returns { hits, method: 'semantic' | 'keyword' }.
+async function rerank(query, hits, opts = {}) {
+  if (!hits || !hits.length) return { hits: hits || [], method: "keyword" };
+  const topK = opts.topK || 20;
+  const cand = hits.slice(0, topK);
+
+  const qv = await embed(query, opts);
+  if (!qv) return { hits, method: "keyword" }; // graceful degrade — the common case
+
+  const scored = [];
+  for (const h of cand) {
+    const hv = await embed((h.title ? h.title + " " : "") + (h.snippet || ""), opts);
+    scored.push({ h, sim: hv ? cosine(qv, hv) : 0 });
+  }
+  scored.sort((a, b) => b.sim - a.sim);
+  const reranked = scored.map((s) => ({ ...s.h, semantic: +s.sim.toFixed(3) })).concat(hits.slice(topK));
+  return { hits: reranked, method: "semantic" };
+}
+
+// C5 — the opt-in gate. Rerank is OFF by default: the caller passes `enabled` (the pull skill
+// sets it from MOVER_OLLAMA). When disabled this is a pure keyword passthrough that NEVER opens
+// a socket — the default path stays zero-network, zero-latency. When enabled, rerank() still
+// gracefully degrades to keyword if Ollama happens to be down. The push hook deliberately does
+// NOT use this (its 1.2s budget can't afford ~21 sequential embeds); it's pull-skill only.
+async function maybeRerank(query, hits, opts = {}) {
+  if (!opts.enabled) return { hits: hits || [], method: "keyword" };
+  return rerank(query, hits, opts);
+}
+
+module.exports = { cosine, embed, rerank, maybeRerank, DEFAULT_HOST, MODEL };

package/src/dashboard/lib/memory-text.js

@@ -0,0 +1,136 @@
+"use strict";
+
+// memory-text.js — shared text utilities for the Mover memory engine.
+// Zero-dep (Node stdlib only). Two distinct tokenizers on purpose:
+//   - tokenize(): SEARCH terms — stopworded, len>=3. Feeds the BM25 index + queries.
+//   - words():    LINK matching — raw lowercased word sequence, NO stopword removal,
+//                 so multi-word note names ("Launch Plan", "The Standing") survive as n-grams.
+// Mixing the two would drop stopword-containing note names and pollute the index with "the".
+
+const fs = require("fs");
+const path = require("path");
+
+// Compact English + Mover-chatter stopword set. Kept inline to stay zero-dep.
+const STOPWORDS = new Set((
+  "a an and are as at be been being but by for from had has have he her hers him his i if in into is it its " +
+  "me my no nor not of off on once only or our ours out over own she so some such than that the their theirs them " +
+  "then there these they this those to too under until up very was we were what when where which while who whom why " +
+  "will with you your yours about above after again against all am any because before below between both did do does " +
+  "doing down during each few more most other same should through too will just dont cant wont youre ive im also get " +
+  "got like one two way thing things stuff lot really actually maybe okay yeah yep nope going want need make made " +
+  "now new use used using via per etc ok let lets going gonna wanna"
+).split(/\s+/).filter(Boolean));
+
+// Generic note-name tokens that are too ambiguous to be a useful wikilink target.
+const VOCAB_STOP = new Set([
+  "index", "readme", "schema", "template", "_template", "misc", "notes", "untitled",
+  "home", "moc", "map", "tools", "people", "places", "scripts", "inbox", "archive",
+  "daily", "weekly", "monthly", "project", "projects", "library", "engine"
+]);
+
+// SEARCH tokens: lowercase, alphanumeric cores (keep internal . - _ for things like
+// "v4.5.1", "engine-writer"), drop stopwords + sub-3-char noise + over-long blobs.
+function tokenize(text) {
+  if (!text) return [];
+  const out = [];
+  const re = /[a-z0-9][a-z0-9._-]{1,40}/gi;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    let t = m[0].toLowerCase().replace(/^[._-]+|[._-]+$/g, "");
+    if (t.length < 3 || t.length > 40) continue;
+    if (STOPWORDS.has(t)) continue;
+    out.push(t);
+  }
+  return out;
+}
+
+// LINK words: lowercase word sequence, light cleanup, NO stopword removal.
+function words(text) {
+  if (!text) return [];
+  const out = [];
+  const re = /[a-z0-9][a-z0-9'._-]*/gi;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const w = m[0].toLowerCase().replace(/^['._-]+|['._-]+$/g, "");
+    if (w) out.push(w);
+  }
+  return out;
+}
+
+// Build uni/bi/tri-gram membership sets from a (capped) text blob, for O(vocab) link matching.
+function ngramSets(text) {
+  const w = words(text);
+  const uni = new Set();
+  const bi = new Set();
+  const tri = new Set();
+  for (let i = 0; i < w.length; i++) {
+    uni.add(w[i]);
+    if (i + 1 < w.length) bi.add(w[i] + " " + w[i + 1]);
+    if (i + 2 < w.length) tri.add(w[i] + " " + w[i + 1] + " " + w[i + 2]);
+  }
+  return { 1: uni, 2: bi, 3: tri };
+}
+
+// term -> frequency map from search tokens.
+function termFreq(tokens) {
+  const tf = Object.create(null);
+  for (const t of tokens) tf[t] = (tf[t] || 0) + 1;
+  return tf;
+}
+
+// Recursively collect .md basenames (no extension) under a dir.
+function listMd(dir, acc, depth) {
+  if (depth < 0) return acc;
+  let ents;
+  try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return acc; }
+  for (const e of ents) {
+    if (e.name.startsWith(".") || e.name === "node_modules") continue;
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) listMd(p, acc, depth - 1);
+    else if (e.name.endsWith(".md")) acc.push(e.name.replace(/\.md$/, ""));
+  }
+  return acc;
+}
+
+// Vocab = the set of existing vault notes the gardener/recall can link into.
+// Sourced from 01_Projects folder names + every .md basename under 03_Library + 02_Areas/Engine.
+// Returns { entries:[{canonical,lower,nWords}], byN:{1:Map,2:Map,3:Map} }.
+function buildVocab(vault) {
+  const raw = new Set();
+  try {
+    for (const e of fs.readdirSync(path.join(vault, "01_Projects"), { withFileTypes: true })) {
+      if (e.isDirectory()) raw.add(e.name);
+    }
+  } catch {}
+  listMd(path.join(vault, "03_Library"), [], 4).forEach((n) => raw.add(n));
+  listMd(path.join(vault, "02_Areas", "Engine"), [], 3).forEach((n) => raw.add(n));
+
+  const entries = [];
+  const byN = { 1: new Map(), 2: new Map(), 3: new Map() };
+  for (const name of raw) {
+    const w = words(name);
+    const nWords = w.length;
+    if (nWords < 1 || nWords > 3) continue;
+    const lower = w.join(" ");
+    // Single-word notes must be >=4 chars and not a generic term, or they match everything.
+    if (nWords === 1 && (lower.length < 4 || VOCAB_STOP.has(lower))) continue;
+    // Skip date-stamped dailies/weeklies AND numbered/ordinal note names ("7 Workflow",
+    // "4. Outbound") — a leading digit is never a useful wikilink target, only index noise.
+    if (/^daily|^weekly|^\d/.test(lower)) continue;
+    if (byN[nWords].has(lower)) continue;
+    byN[nWords].set(lower, name);
+    entries.push({ canonical: name, lower, nWords });
+  }
+  return { entries, byN };
+}
+
+// Given precomputed ngramSets for a session and a vocab, return matched canonical note names.
+function matchLinks(sets, vocab) {
+  const hits = [];
+  for (const e of vocab.entries) {
+    if (sets[e.nWords] && sets[e.nWords].has(e.lower)) hits.push(e.canonical);
+  }
+  return hits;
+}
+
+module.exports = { tokenize, words, ngramSets, termFreq, buildVocab, matchLinks, STOPWORDS, VOCAB_STOP };