march-cli 0.1.41 → 0.1.45

package/src/context/system-core/base.md

@@ -96,7 +96,7 @@ The user primarily asks for software engineering work: fixing bugs, adding behav
 </git_contract>
 
 <memory_system>
-- [recall source="..."] blocks in recent_chat are lightweight recall hints matched from prior thinking output. Treat them as possibly relevant pointers, not as complete facts.
+- [recall] blocks in recent_chat are lightweight memory hints matched by semantic recall. Treat them as possibly relevant pointers, not as complete facts.
 - A recall hint's description may record key operational constraints, including when the full memory must be opened; factor those constraints into relevance before acting.
 - If a recall hint may help the current task, use memory_open(id) to read the full memory before relying on it. Ignore hints that are clearly unrelated or too low-value for the task.
 - Use memory_search(query) for full-text search across all memories.

package/src/memory/markdown/markdown-format.mjs

@@ -32,24 +32,7 @@ export function normalizeTags(tags) {
   return out;
 }
 
-export function expandTags(tags) {
-  const terms = [];
-  for (const tag of tags) {
-    terms.push(tag);
-    for (const part of tag.split(/[\/_-]+/)) {
-      if (part) terms.push(part);
-    }
-  }
-  return [...new Set(terms.map(normalizeText).filter(Boolean))];
-}
 
-export function quoteFtsTerm(term) {
-  return `"${String(term).replace(/"/g, '""')}"`;
-}
-
-export function normalizeText(text) {
-  return String(text ?? "").trim().toLowerCase();
-}
 
 export function generateMemoryId() {
   return `mem_${randomUUID().replace(/-/g, "").slice(0, 16)}`;

package/src/memory/markdown/markdown-recall.mjs

@@ -1,28 +1,20 @@
-import { expandTags, normalizeText } from "./markdown-format.mjs";
-
-export function formatRecallHints(source, hints = []) {
+export function formatRecallHints(hints = []) {
   if (!hints.length) return "";
-  const lines = [`[recall source="${source}"]`];
+  const lines = ["[recall]"];
   for (const hint of hints) {
-    lines.push(`- ${hint.id} | ${hint.name} | ${hint.description}`);
+    lines.push(`- ${hint.id}${formatScoreForPrompt(hint.score)} | ${hint.name} | ${hint.description}`);
   }
   return lines.join("\n");
 }
 
-export function scoreEntry(entry, terms, currentProject) {
-  const expanded = expandTags(entry.tags);
-  let score = 0;
-  for (const term of terms) {
-    if (entry.tags.map(normalizeText).includes(term)) score += 10;
-    else if (expanded.includes(term)) score += 5;
-  }
-  if (currentProject) {
-    const projectTag = normalizeText(`project/${currentProject}`);
-    if (entry.tags.map(normalizeText).includes(projectTag)) score += 2;
-  }
-  return score;
+export function toHint(entry, metadata = {}) {
+  return { id: entry.id, name: entry.name, description: entry.description, ...metadata };
+}
+
+function formatScoreForPrompt(score) {
+  return Number.isFinite(score) ? ` | score=${formatScore(score)}` : "";
 }
 
-export function toHint(entry) {
-  return { id: entry.id, name: entry.name, description: entry.description };
+export function formatScore(score) {
+  return Number.isFinite(score) ? score.toFixed(2) : "--";
 }

package/src/memory/markdown/semantic-preload.mjs

@@ -0,0 +1,17 @@
+export async function preloadSemanticMemoryRecall({ memoryStore, ui = null, logger = null } = {}) {
+  if (!memoryStore?.semanticRecall?.enabled) return { ok: true, skipped: true };
+  try {
+    ui?.status?.("Preparing memory recall model...");
+    await memoryStore.semanticRecall.preload();
+    memoryStore.semanticRecallWarning = memoryStore.semanticRecall.warning;
+    if (memoryStore.semanticRecallWarning) ui?.writeln?.(`Memory recall fallback: ${memoryStore.semanticRecallWarning}`);
+    logger?.event?.("memory.semantic_model_ready", { modelId: memoryStore.semanticRecall.modelId, status: memoryStore.semanticRecall.status });
+    return { ok: true, skipped: false, fallback: memoryStore.semanticRecall.status === "fallback" };
+  } catch (err) {
+    const message = err?.message ?? String(err);
+    memoryStore.semanticRecallWarning = message;
+    logger?.error?.("memory.semantic_model_preload_failed", { error: message });
+    ui?.writeln?.(`Memory recall model preload failed: ${message}`);
+    return { ok: false, error: message };
+  }
+}

package/src/memory/markdown/semantic-recall.mjs

@@ -0,0 +1,165 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Model2VecVectorizer } from "../../agent/code-search/retrieval/model2vec.mjs";
+import { ResilientVectorizer } from "../../agent/code-search/retrieval/resilient-vectorizer.mjs";
+import { parseMemoryMarkdown } from "./markdown-format.mjs";
+
+export const POTION_RETRIEVAL_MODEL_ID = "minishlab/potion-retrieval-32M";
+
+const MAX_CHUNK_CHARS = 1800;
+export const DEFAULT_MEMORY_RECALL_MIN_SCORE = 0.3;
+
+export class SemanticMemoryRecallIndex {
+  constructor({ stateRoot = null, modelId = POTION_RETRIEVAL_MODEL_ID, modelDir = null, vectorizer = null, minScore = parseMemoryRecallMinScore() } = {}) {
+    this.modelId = modelId;
+    this.minScore = minScore;
+    this.vectorizer = vectorizer ?? createDefaultVectorizer({ stateRoot, modelId, modelDir });
+    this.signature = "";
+    this.chunks = [];
+    this.vectors = [];
+  }
+
+  get enabled() {
+    return Boolean(this.vectorizer);
+  }
+
+  get warning() {
+    return this.vectorizer?.warning ?? null;
+  }
+
+  get status() {
+    return this.vectorizer?.status ?? "primary";
+  }
+
+  async preload() {
+    if (!this.vectorizer) return false;
+    if (typeof this.vectorizer.load === "function") await this.vectorizer.load();
+    else await this.vectorizer.encode(["memory recall warmup"]);
+    return true;
+  }
+
+  async search(query, { entries, excluded = new Set(), limit = 3, candidateLimit = 5 } = {}) {
+    const empty = { recalled: [], candidates: [], threshold: this.minScore };
+    if (!this.vectorizer || !String(query ?? "").trim()) return empty;
+    const activeEntries = [...entries.values()].filter((entry) => entry.status === "active" && entry.description && !excluded.has(entry.id));
+    if (activeEntries.length === 0) return empty;
+    await this.#ensureIndex(activeEntries);
+    const [queryVector] = await this.vectorizer.encode([query]);
+    if (!queryVector || queryVector.norm === 0) return empty;
+
+    const bestByEntry = new Map();
+    for (let index = 0; index < this.vectors.length; index += 1) {
+      const chunk = this.chunks[index];
+      if (excluded.has(chunk.entry.id)) continue;
+      const score = cosineSimilarity(queryVector, this.vectors[index]);
+      const prev = bestByEntry.get(chunk.entry.id);
+      if (!prev || score > prev.score) bestByEntry.set(chunk.entry.id, { entry: chunk.entry, score });
+    }
+
+    const candidates = [...bestByEntry.values()]
+      .filter(({ score }) => Number.isFinite(score) && score > 0)
+      .sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name))
+      .map(({ entry, score }) => ({ entry, score, recalled: score >= this.minScore }));
+    return {
+      recalled: candidates.filter((candidate) => candidate.recalled).slice(0, limit),
+      candidates: candidates.slice(0, Math.max(limit, candidateLimit)),
+      threshold: this.minScore,
+      vectorizerStatus: this.status,
+      warning: this.warning,
+    };
+  }
+
+  async #ensureIndex(entries) {
+    const signature = entries.map(entrySignature).join("\n");
+    if (signature === this.signature) return;
+    this.chunks = entries.flatMap(memoryChunks);
+    this.vectors = this.chunks.length > 0
+      ? await this.vectorizer.encode(this.chunks.map((chunk) => chunk.text))
+      : [];
+    this.signature = signature;
+  }
+}
+
+export function parseMemoryRecallMinScore(value = process.env.MARCH_MEMORY_RECALL_MIN_SCORE) {
+  if (value == null || value === "") return DEFAULT_MEMORY_RECALL_MIN_SCORE;
+  const normalized = String(value).trim().toLowerCase();
+  if (["false", "no", "off"].includes(normalized)) return 0;
+  const parsed = Number(normalized);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_MEMORY_RECALL_MIN_SCORE;
+}
+
+function createDefaultVectorizer({ stateRoot, modelId, modelDir }) {
+  const dir = modelDir ?? (stateRoot ? join(stateRoot, "memory", "models", modelId.replaceAll("/", "__")) : null);
+  if (!dir) return null;
+  return new ResilientVectorizer({
+    primary: new Model2VecVectorizer({ modelDir: dir, modelId }),
+    label: "memory recall",
+  });
+}
+
+function memoryChunks(entry) {
+  const body = readMemoryBody(entry);
+  const sections = splitMarkdownBody(body);
+  const chunks = sections.length > 0 ? sections : [""];
+  return chunks.map((section, index) => ({
+    entry,
+    index,
+    text: [
+      entry.name,
+      entry.description,
+      entry.tags.join(" "),
+      section,
+    ].filter(Boolean).join("\n"),
+  }));
+}
+
+function readMemoryBody(entry) {
+  try {
+    return parseMemoryMarkdown(readFileSync(entry.path, "utf8")).body.trim();
+  } catch {
+    return "";
+  }
+}
+
+function splitMarkdownBody(body) {
+  const blocks = body
+    .split(/\n{2,}/)
+    .map((block) => block.trim())
+    .filter(Boolean);
+  const chunks = [];
+  let current = "";
+  for (const block of blocks) {
+    if (!current) {
+      current = block;
+      continue;
+    }
+    if (current.length + block.length + 2 <= MAX_CHUNK_CHARS) {
+      current = `${current}\n\n${block}`;
+      continue;
+    }
+    chunks.push(current);
+    current = block;
+  }
+  if (current) chunks.push(current);
+  return chunks.flatMap(splitOversizedChunk);
+}
+
+function splitOversizedChunk(text) {
+  if (text.length <= MAX_CHUNK_CHARS) return [text];
+  const chunks = [];
+  for (let index = 0; index < text.length; index += MAX_CHUNK_CHARS) {
+    chunks.push(text.slice(index, index + MAX_CHUNK_CHARS));
+  }
+  return chunks;
+}
+
+function entrySignature(entry) {
+  return `${entry.id}:${entry.path}:${Math.trunc(entry.mtimeMs ?? 0)}:${entry.size ?? 0}`;
+}
+
+function cosineSimilarity(left, right) {
+  if (!left?.norm || !right?.norm) return 0;
+  let dot = 0;
+  for (let index = 0; index < left.values.length; index += 1) dot += left.values[index] * right.values[index];
+  return dot / (left.norm * right.norm);
+}

package/src/memory/markdown/sqlite-index.mjs

@@ -15,11 +15,6 @@ CREATE TABLE IF NOT EXISTS memory_index (
   mtime_ms REAL NOT NULL,
   size INTEGER NOT NULL
 );
-CREATE VIRTUAL TABLE IF NOT EXISTS memory_tags_fts USING fts5(
-  id UNINDEXED,
-  tags_text,
-  tokenize = 'unicode61'
-);
 `;
 
 export function openMarkdownMemoryIndex(path) {
@@ -33,7 +28,6 @@ export function openMarkdownMemoryIndex(path) {
 
 export function clearMarkdownMemoryIndex(db) {
   db.exec("DELETE FROM memory_index");
-  db.exec("DELETE FROM memory_tags_fts");
 }
 
 export function loadMarkdownMemoryIndex(db) {
@@ -48,17 +42,15 @@ export function loadMarkdownMemoryIndex(db) {
   return { entries, pathStats };
 }
 
-export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
+export function replaceMarkdownMemoryIndex(db, entries) {
   db.exec("BEGIN IMMEDIATE");
   try {
     clearMarkdownMemoryIndex(db);
     const insertMeta = db.prepare(
       "INSERT INTO memory_index (id, path, name, description, tags_json, status, created_at, updated_at, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
     );
-    const insertFts = db.prepare("INSERT INTO memory_tags_fts (id, tags_text) VALUES (?, ?)");
     for (const entry of entries.values()) {
       insertMeta.run(entry.id, entry.path, entry.name, entry.description, JSON.stringify(entry.tags), entry.status, entry.createdAt, entry.updatedAt, entry.mtimeMs, entry.size);
-      insertFts.run(entry.id, expandTags(entry.tags).join(" "));
     }
     db.exec("COMMIT");
   } catch (err) {
@@ -67,10 +59,6 @@ export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
   }
 }
 
-export function queryMarkdownMemoryIndex(db, query) {
-  return db.prepare("SELECT id FROM memory_tags_fts WHERE tags_text MATCH ? LIMIT 50").all(query);
-}
-
 function rowToEntry(row) {
   return {
     id: String(row.id),

package/src/memory/markdown-store.mjs

@@ -1,17 +1,15 @@
 import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
 import {
-  expandTags,
   formatMemoryMarkdown,
   generateMemoryId,
   normalizeTags,
-  normalizeText,
   parseMemoryMarkdown,
-  quoteFtsTerm,
   walkMarkdownFiles,
 } from "./markdown/markdown-format.mjs";
-import { scoreEntry, toHint } from "./markdown/markdown-recall.mjs";
-import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, queryMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
+import { toHint } from "./markdown/markdown-recall.mjs";
+import { SemanticMemoryRecallIndex } from "./markdown/semantic-recall.mjs";
+import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
 import { softDeleteMemoryFile } from "./markdown/markdown-delete.mjs";
 import { isMemoryIdLike, isSingleEditAway } from "./markdown/memory-id.mjs";
 import { openMarkdownRoot, searchMarkdownRoot } from "./search.mjs";
@@ -22,15 +20,17 @@ export { normalizeTags } from "./markdown/markdown-format.mjs";
 const DEFAULT_SCAN_INTERVAL_MS = 5000;
 
 export class MarkdownMemoryStore {
-  constructor({ root, now = () => new Date(), indexPath = null } = {}) {
+  constructor({ root, now = () => new Date(), indexPath = null, stateRoot = null, semanticRecall = true, semanticVectorizer = null, semanticModelId = undefined, semanticModelDir = null, semanticMinScore = undefined } = {}) {
     if (!root) throw new Error("MarkdownMemoryStore requires a root path");
     this.root = resolve(root);
     this.now = now;
+    this.semanticRecall = semanticRecall ? new SemanticMemoryRecallIndex({ stateRoot, modelId: semanticModelId, modelDir: semanticModelDir, vectorizer: semanticVectorizer, minScore: semanticMinScore }) : null;
+    this.semanticRecallWarning = null;
+    this.lastUserRecallReport = null;
     this.indexPath = indexPath ? resolve(indexPath) : join(this.root, ".march-memory-index.sqlite");
     this.db = openMarkdownMemoryIndex(this.indexPath);
     this.entries = new Map();
     this.pathStats = new Map();
-    this.tagDictionary = new Set();
     this.diagnostics = [];
     this.lastScanAt = 0;
     this.scanIntervalMs = DEFAULT_SCAN_INTERVAL_MS;
@@ -99,8 +99,7 @@ export class MarkdownMemoryStore {
     this.entries = nextEntries;
     this.pathStats = nextStats;
     this.diagnostics = diagnostics;
-    this.#rebuildTagDictionary();
-    replaceMarkdownMemoryIndex(this.db, this.entries, expandTags);
+    replaceMarkdownMemoryIndex(this.db, this.entries);
     this.lastScanAt = Date.now();
     return { entries: this.entries.size, diagnostics };
   }
@@ -121,18 +120,16 @@ export class MarkdownMemoryStore {
     this.db.close?.();
   }
 
-  recallForUser(text, { limit = 3, currentProject = "", excludedIds = [] } = {}) {
+  async recallForUser(text, { limit = 3, excludedIds = [] } = {}) {
     const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
-    const hints = this.#recall(text, { limit, excluded, currentProject });
-    for (const hint of hints) {
-      this.turnSeenMemoryIds.add(hint.id);
-    }
+    const hints = await this.#recallSemantic(text, { limit, excluded });
+    for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
     return hints;
   }
 
-  recallForAssistant(text, { limit = 2, currentProject = "", excludedIds = [] } = {}) {
+  async recallForAssistant(text, { limit = 2, excludedIds = [] } = {}) {
     const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
-    const hints = this.#recall(text, { limit, excluded, currentProject });
+    const hints = await this.#recallSemantic(text, { limit, excluded, recordReport: false });
     for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
     return hints;
   }
@@ -205,46 +202,21 @@ export class MarkdownMemoryStore {
     return result;
   }
 
-  #recall(text, { limit, excluded, currentProject }) {
+  async #recallSemantic(text, { limit, excluded, recordReport = true }) {
     this.ensureFresh();
-    const queryTerms = this.#extractKnownTagTerms(text);
-    if (queryTerms.length === 0) return [];
-    const query = queryTerms.map(quoteFtsTerm).join(" OR ");
-    let rows = [];
+    if (recordReport) this.lastUserRecallReport = null;
+    if (!this.semanticRecall?.enabled) return [];
     try {
-      rows = queryMarkdownMemoryIndex(this.db, query);
-    } catch {
+      const result = await this.semanticRecall.search(text, { entries: this.entries, excluded, limit });
+      const hints = result.recalled.map(({ entry, score }) => toHint(entry, { score }));
+      if (recordReport) {
+        this.lastUserRecallReport = { threshold: result.threshold, vectorizerStatus: result.vectorizerStatus, warning: result.warning, hints, candidates: result.candidates.map(({ entry, score, recalled }) => ({ ...toHint(entry, { score }), recalled })) };
+      }
+      return hints;
+    } catch (err) {
+      this.semanticRecallWarning = err?.message ?? String(err);
       return [];
     }
-    const scored = [];
-    for (const row of rows) {
-      if (excluded.has(row.id)) continue;
-      const entry = this.entries.get(row.id);
-      if (!entry || entry.status !== "active" || !entry.description) continue;
-      const score = scoreEntry(entry, queryTerms, currentProject);
-      if (score <= 0) continue;
-      scored.push({ score, entry });
-    }
-    scored.sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
-    return scored.slice(0, limit).map(({ entry }) => toHint(entry));
-  }
-
-  #extractKnownTagTerms(text) {
-    const normalized = normalizeText(text);
-    if (!normalized) return [];
-    const terms = [];
-    for (const term of this.tagDictionary) {
-      if (term.length < 2) continue;
-      if (normalized.includes(term)) terms.push(term);
-    }
-    return [...new Set(terms)].sort((a, b) => b.length - a.length).slice(0, 16);
-  }
-
-  #rebuildTagDictionary() {
-    this.tagDictionary = new Set();
-    for (const entry of this.entries.values()) {
-      for (const term of expandTags(entry.tags)) this.tagDictionary.add(normalizeText(term));
-    }
   }
 
   #newMemoryPath(isoDate, id) {