march-cli 0.1.42 → 0.1.45

package/src/memory/markdown/semantic-recall.mjs

@@ -0,0 +1,165 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Model2VecVectorizer } from "../../agent/code-search/retrieval/model2vec.mjs";
+import { ResilientVectorizer } from "../../agent/code-search/retrieval/resilient-vectorizer.mjs";
+import { parseMemoryMarkdown } from "./markdown-format.mjs";
+
+export const POTION_RETRIEVAL_MODEL_ID = "minishlab/potion-retrieval-32M";
+
+const MAX_CHUNK_CHARS = 1800;
+export const DEFAULT_MEMORY_RECALL_MIN_SCORE = 0.3;
+
+export class SemanticMemoryRecallIndex {
+  constructor({ stateRoot = null, modelId = POTION_RETRIEVAL_MODEL_ID, modelDir = null, vectorizer = null, minScore = parseMemoryRecallMinScore() } = {}) {
+    this.modelId = modelId;
+    this.minScore = minScore;
+    this.vectorizer = vectorizer ?? createDefaultVectorizer({ stateRoot, modelId, modelDir });
+    this.signature = "";
+    this.chunks = [];
+    this.vectors = [];
+  }
+
+  get enabled() {
+    return Boolean(this.vectorizer);
+  }
+
+  get warning() {
+    return this.vectorizer?.warning ?? null;
+  }
+
+  get status() {
+    return this.vectorizer?.status ?? "primary";
+  }
+
+  async preload() {
+    if (!this.vectorizer) return false;
+    if (typeof this.vectorizer.load === "function") await this.vectorizer.load();
+    else await this.vectorizer.encode(["memory recall warmup"]);
+    return true;
+  }
+
+  async search(query, { entries, excluded = new Set(), limit = 3, candidateLimit = 5 } = {}) {
+    const empty = { recalled: [], candidates: [], threshold: this.minScore };
+    if (!this.vectorizer || !String(query ?? "").trim()) return empty;
+    const activeEntries = [...entries.values()].filter((entry) => entry.status === "active" && entry.description && !excluded.has(entry.id));
+    if (activeEntries.length === 0) return empty;
+    await this.#ensureIndex(activeEntries);
+    const [queryVector] = await this.vectorizer.encode([query]);
+    if (!queryVector || queryVector.norm === 0) return empty;
+
+    const bestByEntry = new Map();
+    for (let index = 0; index < this.vectors.length; index += 1) {
+      const chunk = this.chunks[index];
+      if (excluded.has(chunk.entry.id)) continue;
+      const score = cosineSimilarity(queryVector, this.vectors[index]);
+      const prev = bestByEntry.get(chunk.entry.id);
+      if (!prev || score > prev.score) bestByEntry.set(chunk.entry.id, { entry: chunk.entry, score });
+    }
+
+    const candidates = [...bestByEntry.values()]
+      .filter(({ score }) => Number.isFinite(score) && score > 0)
+      .sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name))
+      .map(({ entry, score }) => ({ entry, score, recalled: score >= this.minScore }));
+    return {
+      recalled: candidates.filter((candidate) => candidate.recalled).slice(0, limit),
+      candidates: candidates.slice(0, Math.max(limit, candidateLimit)),
+      threshold: this.minScore,
+      vectorizerStatus: this.status,
+      warning: this.warning,
+    };
+  }
+
+  async #ensureIndex(entries) {
+    const signature = entries.map(entrySignature).join("\n");
+    if (signature === this.signature) return;
+    this.chunks = entries.flatMap(memoryChunks);
+    this.vectors = this.chunks.length > 0
+      ? await this.vectorizer.encode(this.chunks.map((chunk) => chunk.text))
+      : [];
+    this.signature = signature;
+  }
+}
+
+export function parseMemoryRecallMinScore(value = process.env.MARCH_MEMORY_RECALL_MIN_SCORE) {
+  if (value == null || value === "") return DEFAULT_MEMORY_RECALL_MIN_SCORE;
+  const normalized = String(value).trim().toLowerCase();
+  if (["false", "no", "off"].includes(normalized)) return 0;
+  const parsed = Number(normalized);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_MEMORY_RECALL_MIN_SCORE;
+}
+
+function createDefaultVectorizer({ stateRoot, modelId, modelDir }) {
+  const dir = modelDir ?? (stateRoot ? join(stateRoot, "memory", "models", modelId.replaceAll("/", "__")) : null);
+  if (!dir) return null;
+  return new ResilientVectorizer({
+    primary: new Model2VecVectorizer({ modelDir: dir, modelId }),
+    label: "memory recall",
+  });
+}
+
+function memoryChunks(entry) {
+  const body = readMemoryBody(entry);
+  const sections = splitMarkdownBody(body);
+  const chunks = sections.length > 0 ? sections : [""];
+  return chunks.map((section, index) => ({
+    entry,
+    index,
+    text: [
+      entry.name,
+      entry.description,
+      entry.tags.join(" "),
+      section,
+    ].filter(Boolean).join("\n"),
+  }));
+}
+
+function readMemoryBody(entry) {
+  try {
+    return parseMemoryMarkdown(readFileSync(entry.path, "utf8")).body.trim();
+  } catch {
+    return "";
+  }
+}
+
+function splitMarkdownBody(body) {
+  const blocks = body
+    .split(/\n{2,}/)
+    .map((block) => block.trim())
+    .filter(Boolean);
+  const chunks = [];
+  let current = "";
+  for (const block of blocks) {
+    if (!current) {
+      current = block;
+      continue;
+    }
+    if (current.length + block.length + 2 <= MAX_CHUNK_CHARS) {
+      current = `${current}\n\n${block}`;
+      continue;
+    }
+    chunks.push(current);
+    current = block;
+  }
+  if (current) chunks.push(current);
+  return chunks.flatMap(splitOversizedChunk);
+}
+
+function splitOversizedChunk(text) {
+  if (text.length <= MAX_CHUNK_CHARS) return [text];
+  const chunks = [];
+  for (let index = 0; index < text.length; index += MAX_CHUNK_CHARS) {
+    chunks.push(text.slice(index, index + MAX_CHUNK_CHARS));
+  }
+  return chunks;
+}
+
+function entrySignature(entry) {
+  return `${entry.id}:${entry.path}:${Math.trunc(entry.mtimeMs ?? 0)}:${entry.size ?? 0}`;
+}
+
+function cosineSimilarity(left, right) {
+  if (!left?.norm || !right?.norm) return 0;
+  let dot = 0;
+  for (let index = 0; index < left.values.length; index += 1) dot += left.values[index] * right.values[index];
+  return dot / (left.norm * right.norm);
+}

package/src/memory/markdown/sqlite-index.mjs

@@ -15,11 +15,6 @@ CREATE TABLE IF NOT EXISTS memory_index (
   mtime_ms REAL NOT NULL,
   size INTEGER NOT NULL
 );
-CREATE VIRTUAL TABLE IF NOT EXISTS memory_tags_fts USING fts5(
-  id UNINDEXED,
-  tags_text,
-  tokenize = 'unicode61'
-);
 `;
 
 export function openMarkdownMemoryIndex(path) {
@@ -33,7 +28,6 @@ export function openMarkdownMemoryIndex(path) {
 
 export function clearMarkdownMemoryIndex(db) {
   db.exec("DELETE FROM memory_index");
-  db.exec("DELETE FROM memory_tags_fts");
 }
 
 export function loadMarkdownMemoryIndex(db) {
@@ -48,17 +42,15 @@ export function loadMarkdownMemoryIndex(db) {
   return { entries, pathStats };
 }
 
-export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
+export function replaceMarkdownMemoryIndex(db, entries) {
   db.exec("BEGIN IMMEDIATE");
   try {
     clearMarkdownMemoryIndex(db);
     const insertMeta = db.prepare(
       "INSERT INTO memory_index (id, path, name, description, tags_json, status, created_at, updated_at, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
     );
-    const insertFts = db.prepare("INSERT INTO memory_tags_fts (id, tags_text) VALUES (?, ?)");
     for (const entry of entries.values()) {
       insertMeta.run(entry.id, entry.path, entry.name, entry.description, JSON.stringify(entry.tags), entry.status, entry.createdAt, entry.updatedAt, entry.mtimeMs, entry.size);
-      insertFts.run(entry.id, expandTags(entry.tags).join(" "));
     }
     db.exec("COMMIT");
   } catch (err) {
@@ -67,10 +59,6 @@ export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
   }
 }
 
-export function queryMarkdownMemoryIndex(db, query) {
-  return db.prepare("SELECT id FROM memory_tags_fts WHERE tags_text MATCH ? LIMIT 50").all(query);
-}
-
 function rowToEntry(row) {
   return {
     id: String(row.id),

package/src/memory/markdown-store.mjs

@@ -1,17 +1,15 @@
 import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
 import {
-  expandTags,
   formatMemoryMarkdown,
   generateMemoryId,
   normalizeTags,
-  normalizeText,
   parseMemoryMarkdown,
-  quoteFtsTerm,
   walkMarkdownFiles,
 } from "./markdown/markdown-format.mjs";
-import { scoreEntry, toHint } from "./markdown/markdown-recall.mjs";
-import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, queryMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
+import { toHint } from "./markdown/markdown-recall.mjs";
+import { SemanticMemoryRecallIndex } from "./markdown/semantic-recall.mjs";
+import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
 import { softDeleteMemoryFile } from "./markdown/markdown-delete.mjs";
 import { isMemoryIdLike, isSingleEditAway } from "./markdown/memory-id.mjs";
 import { openMarkdownRoot, searchMarkdownRoot } from "./search.mjs";
@@ -22,15 +20,17 @@ export { normalizeTags } from "./markdown/markdown-format.mjs";
 const DEFAULT_SCAN_INTERVAL_MS = 5000;
 
 export class MarkdownMemoryStore {
-  constructor({ root, now = () => new Date(), indexPath = null } = {}) {
+  constructor({ root, now = () => new Date(), indexPath = null, stateRoot = null, semanticRecall = true, semanticVectorizer = null, semanticModelId = undefined, semanticModelDir = null, semanticMinScore = undefined } = {}) {
     if (!root) throw new Error("MarkdownMemoryStore requires a root path");
     this.root = resolve(root);
     this.now = now;
+    this.semanticRecall = semanticRecall ? new SemanticMemoryRecallIndex({ stateRoot, modelId: semanticModelId, modelDir: semanticModelDir, vectorizer: semanticVectorizer, minScore: semanticMinScore }) : null;
+    this.semanticRecallWarning = null;
+    this.lastUserRecallReport = null;
     this.indexPath = indexPath ? resolve(indexPath) : join(this.root, ".march-memory-index.sqlite");
     this.db = openMarkdownMemoryIndex(this.indexPath);
     this.entries = new Map();
     this.pathStats = new Map();
-    this.tagDictionary = new Set();
     this.diagnostics = [];
     this.lastScanAt = 0;
     this.scanIntervalMs = DEFAULT_SCAN_INTERVAL_MS;
@@ -99,8 +99,7 @@ export class MarkdownMemoryStore {
     this.entries = nextEntries;
     this.pathStats = nextStats;
     this.diagnostics = diagnostics;
-    this.#rebuildTagDictionary();
-    replaceMarkdownMemoryIndex(this.db, this.entries, expandTags);
+    replaceMarkdownMemoryIndex(this.db, this.entries);
     this.lastScanAt = Date.now();
     return { entries: this.entries.size, diagnostics };
   }
@@ -121,18 +120,16 @@ export class MarkdownMemoryStore {
     this.db.close?.();
   }
 
-  recallForUser(text, { limit = 3, currentProject = "", excludedIds = [] } = {}) {
+  async recallForUser(text, { limit = 3, excludedIds = [] } = {}) {
     const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
-    const hints = this.#recall(text, { limit, excluded, currentProject });
-    for (const hint of hints) {
-      this.turnSeenMemoryIds.add(hint.id);
-    }
+    const hints = await this.#recallSemantic(text, { limit, excluded });
+    for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
     return hints;
   }
 
-  recallForAssistant(text, { limit = 2, currentProject = "", excludedIds = [] } = {}) {
+  async recallForAssistant(text, { limit = 2, excludedIds = [] } = {}) {
     const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
-    const hints = this.#recall(text, { limit, excluded, currentProject });
+    const hints = await this.#recallSemantic(text, { limit, excluded, recordReport: false });
     for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
     return hints;
   }
@@ -205,46 +202,21 @@ export class MarkdownMemoryStore {
     return result;
   }
 
-  #recall(text, { limit, excluded, currentProject }) {
+  async #recallSemantic(text, { limit, excluded, recordReport = true }) {
     this.ensureFresh();
-    const queryTerms = this.#extractKnownTagTerms(text);
-    if (queryTerms.length === 0) return [];
-    const query = queryTerms.map(quoteFtsTerm).join(" OR ");
-    let rows = [];
+    if (recordReport) this.lastUserRecallReport = null;
+    if (!this.semanticRecall?.enabled) return [];
     try {
-      rows = queryMarkdownMemoryIndex(this.db, query);
-    } catch {
+      const result = await this.semanticRecall.search(text, { entries: this.entries, excluded, limit });
+      const hints = result.recalled.map(({ entry, score }) => toHint(entry, { score }));
+      if (recordReport) {
+        this.lastUserRecallReport = { threshold: result.threshold, vectorizerStatus: result.vectorizerStatus, warning: result.warning, hints, candidates: result.candidates.map(({ entry, score, recalled }) => ({ ...toHint(entry, { score }), recalled })) };
+      }
+      return hints;
+    } catch (err) {
+      this.semanticRecallWarning = err?.message ?? String(err);
       return [];
     }
-    const scored = [];
-    for (const row of rows) {
-      if (excluded.has(row.id)) continue;
-      const entry = this.entries.get(row.id);
-      if (!entry || entry.status !== "active" || !entry.description) continue;
-      const score = scoreEntry(entry, queryTerms, currentProject);
-      if (score <= 0) continue;
-      scored.push({ score, entry });
-    }
-    scored.sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
-    return scored.slice(0, limit).map(({ entry }) => toHint(entry));
-  }
-
-  #extractKnownTagTerms(text) {
-    const normalized = normalizeText(text);
-    if (!normalized) return [];
-    const terms = [];
-    for (const term of this.tagDictionary) {
-      if (term.length < 2) continue;
-      if (normalized.includes(term)) terms.push(term);
-    }
-    return [...new Set(terms)].sort((a, b) => b.length - a.length).slice(0, 16);
-  }
-
-  #rebuildTagDictionary() {
-    this.tagDictionary = new Set();
-    for (const entry of this.entries.values()) {
-      for (const term of expandTags(entry.tags)) this.tagDictionary.add(normalizeText(term));
-    }
   }
 
   #newMemoryPath(isoDate, id) {