claude-recall 0.37.2 → 0.38.0

package/README.md

@@ -464,6 +464,7 @@ Defaults work out of the box; tune via environment variables as needed.
 | `CLAUDE_RECALL_LLM_TIMEOUT_MS`           | `5000`  | Timeout for hook-context LLM calls (classification, hindsight hints). Hooks fall back to regex when it fires. |
 | `CLAUDE_RECALL_STOP_DEBOUNCE_MS`         | `300000` | Debounce for the heavy Stop-hook pipeline (episodes, session extraction, promotion). `0` disables. |
 | `CLAUDE_RECALL_PROJECT_ID`               | *(cwd)*  | Pin the project scope to a fixed id, overriding working-directory detection. |
+| `CLAUDE_RECALL_RETRIEVAL`                | `like`  | Lexical retrieval engine: `fts` uses SQLite FTS5 / BM25 ranking for better paraphrase recall; `like` (default) uses the legacy substring filter. Opt-in; falls back to `like` automatically if the SQLite build lacks FTS5. See [docs/design-hybrid-retrieval-fts5.md](docs/design-hybrid-retrieval-fts5.md). |
 
 ---
 

package/dist/core/retrieval.js

@@ -102,8 +102,17 @@ class MemoryRetrieval {
     }
     calculateRelevance(memory, context, stats, evidenceCount) {
         let score = memory.relevance_score || 1.0;
-        // Boost for keyword matches in memory value
-        if (context.keywords && context.keywords.length > 0) {
+        // Lexical boost. Two sources, mutually exclusive:
+        //   (a) FTS5 path — storage attached a normalized bm25Score ∈ [0,1]; every
+        //       returned candidate already matched the query (MATCH filtered), so
+        //       there is no "no-overlap" case here.
+        //   (b) LIKE path — no bm25Score; fall back to keyword-overlap counting.
+        if (memory.bm25Score !== undefined) {
+            // Fuse BM25 multiplicatively, preserving the LIKE path's dynamic range
+            // (~1x..4x) so decay/strength/evidence boosts behave identically.
+            score *= 1 + MemoryRetrieval.W_LEXICAL * memory.bm25Score;
+        }
+        else if (context.keywords && context.keywords.length > 0) {
             const memoryStr = JSON.stringify(memory.value).toLowerCase();
             let keywordMatches = 0;
             for (const keyword of context.keywords) {
@@ -268,6 +277,13 @@ class MemoryRetrieval {
     }
 }
 exports.MemoryRetrieval = MemoryRetrieval;
+/**
+ * Fusion weight for the FTS5 BM25 lexical signal. Chosen so a top match
+ * (bm25Score=1) yields a ~4x boost, matching the LIKE path's full-match
+ * dynamic range (1 + matchRatio*3, ×1.5 all-match). Tune against the
+ * retrieval benchmark before flipping CLAUDE_RECALL_RETRIEVAL=fts to default.
+ */
+MemoryRetrieval.W_LEXICAL = 3.0;
 MemoryRetrieval.TYPE_PRIORITY = {
     'correction': 6,
     'solution': 5.5, // hard-won reusable solutions — high signal, rank just below corrections

package/dist/memory/storage.js

@@ -44,6 +44,13 @@ const path = __importStar(require("path"));
 const test_pollution_1 = require("../services/test-pollution");
 class MemoryStorage {
     constructor(dbPath) {
+        /**
+         * Whether the FTS5 mirror table + triggers are present and usable. Set once
+         * during setupFts(); when false, searchByContext always uses the LIKE path
+         * regardless of retrievalMode (self-built/exotic SQLite may lack FTS5).
+         */
+        this.ftsAvailable = false;
+        this.retrievalMode = (process.env.CLAUDE_RECALL_RETRIEVAL || 'like').trim().toLowerCase();
         this.db = new better_sqlite3_1.default(dbPath);
         // Enable WAL mode for better concurrency and to ensure writes are visible
         this.db.pragma('journal_mode = WAL');
@@ -250,6 +257,85 @@ class MemoryStorage {
             console.error('⚠️  Schema migration error:', error);
             // Don't throw - let the database continue with existing schema
         }
+        // FTS5 lexical index (v0.38.0+). Separate from the try block above so a
+        // FTS5-less SQLite build degrades to the LIKE path instead of aborting the
+        // whole migration.
+        this.setupFts();
+    }
+    /**
+     * Create the FTS5 mirror of memories.value (external-content table kept in
+     * sync by triggers) and backfill it once. Feature-detected: if this SQLite
+     * build lacks FTS5, ftsAvailable stays false and retrieval uses LIKE.
+     *
+     * The table + triggers are derived, redundant data — dropping them reverts to
+     * LIKE with zero risk to `memories`. Triggers do the syncing in SQL so every
+     * writer (upsert, INSERT OR REPLACE, delete, import) stays covered without a
+     * TypeScript write-path change.
+     */
+    setupFts() {
+        try {
+            // Was the mirror already present before this startup? If so, the triggers
+            // below have been maintaining it and we must NOT re-backfill. We cannot
+            // gauge this from `count(*) FROM memories_fts`: for an external-content
+            // table that count proxies to the content table, so it reads non-zero
+            // even when the index is empty (the legacy-upgrade bug).
+            const existed = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memories_fts'").get();
+            this.db.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+          value,
+          content='memories',
+          content_rowid='id'
+        );
+        CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+          INSERT INTO memories_fts(rowid, value) VALUES (new.id, new.value);
+        END;
+        CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+          INSERT INTO memories_fts(memories_fts, rowid, value) VALUES('delete', old.id, old.value);
+        END;
+        CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+          INSERT INTO memories_fts(memories_fts, rowid, value) VALUES('delete', old.id, old.value);
+          INSERT INTO memories_fts(rowid, value) VALUES (new.id, new.value);
+        END;
+      `);
+            // First time the mirror is created (fresh DB or legacy upgrade): backfill
+            // the index from existing rows via FTS5's canonical 'rebuild' command —
+            // the correct way to populate an external-content index from content. A
+            // single scan, trivial at the 10k row cap. Skipped on later startups.
+            if (!existed) {
+                this.db.exec("INSERT INTO memories_fts(memories_fts) VALUES('rebuild')");
+            }
+            this.ftsAvailable = true;
+        }
+        catch (error) {
+            // FTS5 unavailable (self-built SQLite without the extension) — leave the
+            // flag off; searchByContext uses the LIKE path everywhere.
+            this.ftsAvailable = false;
+        }
+    }
+    /**
+     * Turn extracted keywords into a safe FTS5 MATCH expression: each term is
+     * stripped to word characters, wrapped as a quoted prefix token, and
+     * OR-joined — e.g. `"kaggle"* OR "submission"*`. Prefix (`*`) mimics the
+     * substring reach of the old LIKE filter ("auth" still matches
+     * "authentication"). Returns '' when nothing usable remains, so the caller
+     * falls back to LIKE rather than issuing an empty MATCH.
+     *
+     * Sanitization is mandatory: bare AND/OR/NEAR, quotes, hyphens and `*` are
+     * FTS5 operators and throw SQLITE_ERROR on malformed input.
+     */
+    sanitizeFtsMatch(keywords) {
+        const terms = [];
+        for (const kw of keywords) {
+            // Keep only letters/digits/space; collapse everything else (quotes,
+            // hyphens, parens, operators) to spaces, then take the first token.
+            const cleaned = String(kw).toLowerCase().replace(/[^a-z0-9\s]/g, ' ').trim();
+            if (!cleaned)
+                continue;
+            const token = cleaned.split(/\s+/)[0];
+            if (token)
+                terms.push(`"${token}"*`);
+        }
+        return [...new Set(terms)].join(' OR ');
     }
     /**
      * Compute a SHA-256 content hash from the meaningful fields of a memory.
@@ -548,6 +634,24 @@ class MemoryStorage {
             throw new Error('searchByContext requires context.project_id or context.includeAllProjects=true. ' +
                 'Calling without either would silently return memories from all projects.');
         }
+        // FTS5 lexical path (opt-in). Only when a keyword filter would otherwise be
+        // applied — empty/stopword queries keep the LIKE branch's "return all
+        // scoped rows" behaviour so load_rules-style calls are unaffected.
+        if (this.retrievalMode === 'fts' &&
+            this.ftsAvailable &&
+            context.keywords &&
+            context.keywords.length > 0) {
+            const matchExpr = this.sanitizeFtsMatch(context.keywords);
+            if (matchExpr) {
+                try {
+                    return this.searchByContextFts(context, matchExpr);
+                }
+                catch {
+                    // Malformed MATCH or FTS error — never crash retrieval; fall through
+                    // to the LIKE path below.
+                }
+            }
+        }
         let query = 'SELECT * FROM memories WHERE 1=1';
         const params = [];
         if (context.project_id) {
@@ -595,6 +699,48 @@ class MemoryStorage {
         const rows = stmt.all(...params);
         return rows.map(row => this.rowToMemory(row));
     }
+    /**
+     * FTS5 candidate fetch: MATCH replaces the LIKE keyword filter while every
+     * other predicate (scope, file_path, type) is preserved verbatim. Attaches a
+     * normalized bm25Score ∈ [0,1] to each row (1 = best match in this set) for
+     * the retrieval-layer fusion. Throws on a malformed MATCH — the caller
+     * catches and falls back to LIKE.
+     */
+    searchByContextFts(context, matchExpr) {
+        let query = `SELECT m.*, bm25(memories_fts) AS bm25_rank
+       FROM memories_fts
+       JOIN memories m ON m.id = memories_fts.rowid
+       WHERE memories_fts MATCH ?`;
+        const params = [matchExpr];
+        if (context.project_id) {
+            query += ' AND (m.project_id = ? OR m.scope = ? OR m.project_id IS NULL)';
+            params.push(context.project_id, 'universal');
+        }
+        if (context.file_path) {
+            query += ' AND m.file_path = ?';
+            params.push(context.file_path);
+        }
+        if (context.type) {
+            query += ' AND m.type = ?';
+            params.push(context.type);
+        }
+        // SQLite bm25() is negative; more-negative = better, so ascending is best-first.
+        query += ' ORDER BY bm25_rank';
+        const rows = this.db.prepare(query).all(...params);
+        if (rows.length === 0)
+            return [];
+        // Min-max normalize -bm25 (so higher = better) across the candidate set.
+        const raws = rows.map(r => -r.bm25_rank);
+        const min = Math.min(...raws);
+        const max = Math.max(...raws);
+        const span = max - min;
+        return rows.map((row, i) => {
+            const memory = this.rowToMemory(row);
+            // Degenerate set (single row, or all equally ranked) → all are the best match.
+            memory.bm25Score = span > 0 ? (raws[i] - min) / span : 1.0;
+            return memory;
+        });
+    }
     deleteByKey(key) {
         const stmt = this.db.prepare('DELETE FROM memories WHERE key = ?');
         const result = stmt.run(key);

package/docs/comparison-mem0.md

@@ -0,0 +1,182 @@
+# Claude Recall vs. mem0
+
+A positioning and architecture comparison against [mem0](https://github.com/mem0ai/mem0)
+(mem0ai), currently the most popular open-source agent-memory project
+(~63k GitHub stars). Written 2026-08-17.
+
+Both give AI agents persistent memory across conversations, but they are built
+for **different consumers** and on **opposite philosophies**: mem0 is an
+LLM-native memory *backend for app developers*; Claude Recall is a *purpose-built,
+zero-dependency memory tool for Claude Code*.
+
+## At a glance
+
+| Dimension | **mem0** | **Claude Recall** |
+|---|---|---|
+| Primary consumer | App developers embedding memory in their own agents | Claude Code users (the coding agent itself) |
+| Delivery | Python/TS SDK + REST API + hosted Platform; optional MCP servers | MCP server + Claude Code hooks, installed via npm |
+| Storage | Vector DB (Qdrant default) + optional graph DB | Local SQLite (WAL mode), single file `~/.claude-recall/claude-recall.db` |
+| Retrieval | Embedding / semantic vector search (top-k) | Keyword ranking + time-decay + context/usage scoring |
+| Extraction | **LLM-driven** — LLM extracts facts, then ADD/UPDATE/DELETE/NOOP reconciliation | Rule/heuristic extractors (preferences, failures, patterns); no LLM in the hot injection path |
+| Hard dependencies | **LLM API key + embeddings pipeline** (defaults to OpenAI) | None — no API key, no embeddings, no external services |
+| Cost per write | LLM tokens + latency on every memory write | ~0 (local compute, ~10–30 ms keyword rank) |
+| Scoping | `user_id` / `agent_id` / `run_id` (session) | Project-scoped (universal vs. project-specific), local machine |
+| License / lang | Apache 2.0; Python + TS | TypeScript |
+| Maturity | ~63k stars; commercial hosted Platform | Early-stage OSS on npm |
+
+## Where they genuinely differ
+
+**1. LLM-native vs. deterministic.** This is the core split. mem0 runs an LLM on
+*every write*: it extracts salient facts, then (classically) does a second LLM
+pass that decides whether to **ADD / UPDATE / DELETE / NOOP** against
+semantically-similar existing memories. That buys real **consolidation and
+contradiction resolution** — but costs tokens, latency, and an API key. Claude
+Recall is deterministic: SHA-256 content-hash dedup at write time, keyword +
+decay + usage ranking at read time, and the just-in-time rule-injector runs *no*
+LLM in the hot path. Recall trades mem0's semantic smarts for **zero marginal
+cost and no network dependency**.
+
+**2. Retrieval mechanics.** mem0 embeds the query and returns top-k semantically
+similar memories — it will match "how do I deploy" to a memory phrased "release
+process" even with no shared words. Recall's keyword ranking
+(`src/core/retrieval.ts`) will not catch that paraphrase. This is exactly the gap
+`docs/design-hybrid-retrieval-fts5.md` targets — SQLite FTS5 narrows it without
+adding an embeddings dependency.
+
+**3. Who does the remembering.** mem0 is a **library you call** —
+`client.add(messages, user_id=...)` / `client.search(query, ...)`. Your app
+decides when to write and read. Claude Recall **inserts itself into the agent's
+loop** via Claude Code hooks: the rule-injector fires on PreToolUse and pushes
+matching rules adjacent to the action, the enforcer gates tool use until rules
+load, PreCompact preserves memory before compaction. mem0 is passive
+infrastructure; Recall is an active participant in the session.
+
+**4. Multi-tenant vs. single-user-local.** mem0 is built for many users on shared
+infrastructure (`user_id`/`agent_id`/`run_id`, hosted cloud). Recall is
+single-developer, single-machine, project-scoped — memories never leave
+`~/.claude-recall/`. That is a privacy/simplicity win and a "no sync across your
+machines" limitation.
+
+## The overlap worth noting
+
+mem0 has two MCP paths onto coding agents: the **Platform MCP** (cloud) and
+**OpenMemory MCP** — a *local-first, self-hosted* MCP server that auto-captures
+coding preferences and injects relevant memories into any MCP agent.
+**OpenMemory is mem0's closest analog to Claude Recall** and the most direct
+competitor. Key contrast even there: OpenMemory still runs the LLM extraction
+pipeline (needs a key); Recall does not.
+
+## Ideas Recall could borrow
+
+- **LLM-based reconcile pass** (optional, background, off the hot path) —
+  mem0's ADD/UPDATE/DELETE/NOOP is stronger dedup than content-hash equality
+  (catches "same fact, different words").
+- **Semantic retrieval** — the FTS5 design note is the pragmatic middle ground;
+  a full embeddings option would close the paraphrase gap but breaks the
+  no-API-key promise.
+- **Temporal/relationship reasoning** — mem0's graph layer wins on multi-hop and
+  "what changed over time" queries. Recall's preference-versioning
+  (`superseded_by`) is a lighter take on the same idea. See the assessment below.
+
+## Bottom line
+
+mem0 optimizes for *semantic recall quality and multi-tenant scale*, accepting
+LLM cost and an API-key dependency. Claude Recall optimizes for *zero-dependency,
+zero-cost, deep Claude Code integration*, accepting weaker paraphrase matching.
+They are not really competitors except at the OpenMemory-MCP boundary — Recall's
+differentiator is that it needs nothing but Node and a local SQLite file, and it
+lives *inside* the agent's action loop rather than beside it.
+
+---
+
+# Assessment: should Recall adopt mem0's graph-memory approach?
+
+## What mem0's graph layer actually is
+
+mem0's graph memory stores an **entity-relationship graph alongside** the vector
+store. On each write, an LLM pipeline runs: **entity extraction → relationship
+establishment → conflict detection → graph update** (with an update resolver for
+temporal conflicts). Retrieval can then traverse relationships, not just match
+text. Reported payoff on LOCOMO: the graph variant scores ~2% higher overall than
+the base config, but the base-vs-graph gains concentrate where it matters —
+**+29.3 on temporal reasoning and +25.2 on multi-hop** questions ("what did I
+decide about X *after* I changed Y?").
+
+Two caveats that reshape the "adopt it" question:
+- In the **v3 OSS rewrite (~Apr 2026)** mem0 **removed the graph layer from
+  open source** and replaced it with lighter **spaCy-based entity linking**
+  (entities stored in a parallel vector collection). Full graph memory now lives
+  on the **hosted Platform**. So even mem0 concluded the LLM-driven graph was too
+  heavy for the embedded/OSS tier.
+- The graph's value is realized through **traversal queries** — a class of query
+  Recall does not currently ask.
+
+## How this maps onto Recall today
+
+Recall's schema (`src/memory/schema.sql`) is a single flat `memories` table.
+There is exactly one relationship edge modeled: preference versioning
+(`superseded_by` / `superseded_at` / `is_active`) — effectively a one-hop
+"replaced-by" chain. Retrieval (`src/core/retrieval.ts`) is pure per-memory
+scoring: keyword overlap × time-decay × project/file match × usage strength ×
+evidence. There is **no traversal, no entity model, no cross-memory linking**.
+
+So adopting graph memory is not a tweak — it means adding (a) an entity/relation
+data model, (b) an extraction step to populate it, and (c) traversal-aware
+retrieval to consume it. Each has a cost.
+
+### The extraction problem is the blocker
+
+mem0's graph quality comes from an **LLM** reading each memory and emitting
+`(subject, relation, object)` triples with conflict resolution. Recall's entire
+design premise is **no LLM and no API key in the pipeline**. To match mem0 you
+would either:
+- **Add an LLM dependency** → breaks the zero-dependency promise, adds cost and
+  latency to every write, needs a key. Non-starter for the core product.
+- **Use spaCy/NER** (mem0's own v3 fallback) → a heavy Python/native dependency
+  in a Node project, and NER on short dev-preference snippets ("commit with
+  `--no-gpg-sign`", "merge before next PR") yields sparse, low-value entities.
+  The corpus is imperative rules, not narrative prose with rich named entities.
+- **Regex/heuristic entity tagging** → cheap and dependency-free, but produces a
+  thin graph that mostly re-encodes what keyword scoring already captures.
+
+### The demand problem
+
+The graph pays off on **multi-hop and temporal** questions. Recall's actual
+retrieval trigger is "surface rules relevant to *this tool call*" — a
+single-hop, relevance-ranked lookup fired by the rule-injector. It does not ask
+"trace the chain of decisions about auth across sessions." Until there is a
+consumer that issues traversal queries, a graph is infrastructure with no reader.
+
+## Verdict: **not worth a full graph layer — adopt the one idea that fits**
+
+A mem0-style LLM-driven knowledge graph is **over-engineered for Recall's corpus,
+consumers, and constraints**, and mem0 itself pulled it out of OSS for the same
+weight reasons. Recommendation, in priority order:
+
+1. **Do first — semantic retrieval via FTS5** (already scoped in
+   `docs/design-hybrid-retrieval-fts5.md`). This solves the *real* observed gap
+   (paraphrase matching) at a fraction of the cost, no new dependency. Higher ROI
+   than any graph work.
+
+2. **Cheap, high-value slice of the graph idea — generalize `superseded_by` into
+   a lightweight typed-link table.** Recall already models one edge type. A small
+   `memory_links(from_key, to_key, relation)` table — relations like
+   `supersedes`, `refines`, `contradicts`, `depends-on` — captures ~80% of the
+   temporal/versioning value with:
+   - no LLM (links come from existing signals: the promotion engine already
+     knows evidence chains; preference override already knows supersession;
+     content-hash near-misses can suggest `refines`),
+   - no new runtime dependency (one SQLite table + indexes),
+   - a bounded blast radius (retrieval can optionally expand one hop from a
+     top-ranked memory, off the hot path).
+
+   This gives "show me the current rule *and what it replaced*" and "this failure
+   plus its fix" without the extraction machinery.
+
+3. **Do not build** — full entity-relationship extraction, graph DB backend
+   (Neo4j/Kuzu/etc.), or LLM triple extraction. Wrong cost/benefit for a
+   local, zero-dependency, single-developer tool.
+
+**One-line recommendation:** skip the graph *layer*; ship FTS5 first, then, if a
+traversal consumer emerges, add a single `memory_links` table that generalizes
+the existing `superseded_by` edge — not an LLM knowledge graph.

package/docs/design-hybrid-retrieval-fts5.md

@@ -0,0 +1,207 @@
+# Design note: FTS5 / BM25 hybrid retrieval
+
+**Status:** ✅ implemented behind `CLAUDE_RECALL_RETRIEVAL=fts` (default `like`).
+PRs 1 & 3 of §10 shipped together — vtable + triggers + feature-detect +
+backfill, and the MATCH candidate fetch + sanitization + BM25 fusion. Still open:
+the benchmark harness (PR 2) and flipping the default (PR 4), which is gated on
+that benchmark showing a win.
+**Date:** 2026-07-22 (proposal) · implemented 2026-08-17 · **Baseline:** v0.37.2
+**Scope of this note:** lexical BM25 ranking via SQLite FTS5. Local embeddings /
+semantic similarity are a deliberately-separate later phase (§9).
+
+**Implementation notes (deltas from the proposal below):**
+- FTS terms are wrapped as **prefix** tokens (`"auth"*`), not plain phrases, so a
+  short keyword still reaches a longer token (`auth` → `authentication`) —
+  preserving the LIKE `%kw%` substring reach. Sanitizer:
+  `MemoryStorage.sanitizeFtsMatch` (`src/memory/storage.ts`).
+- Backfill uses FTS5's canonical `INSERT INTO memories_fts(memories_fts)
+  VALUES('rebuild')`, gated on whether the vtable **existed before startup** (via
+  `sqlite_master`). The proposal's "only if the FTS table is empty" guard is
+  **wrong for external-content tables** — `count(*)` there proxies to the content
+  table and reads non-zero even with an empty index (would silently skip the
+  legacy-upgrade backfill; covered by a regression test).
+- Fusion is `score *= 1 + W_LEXICAL * bm25Score` with `W_LEXICAL = 3.0`
+  (`src/core/retrieval.ts`); `bm25Score ∈ [0,1]` is min-max normalized in
+  `MemoryStorage.searchByContextFts` and carried on `Memory.bm25Score`.
+- Tests: `tests/unit/storage-fts.test.ts` (7 cases: match+score, prefix recall,
+  scope isolation, trigger sync on update/delete, sanitize-to-empty safety,
+  legacy backfill, and no-bm25Score-on-`like`).
+
+---
+
+## 1. Goal
+
+Replace the crude `LIKE '%kw%'` candidate filter + `includes()` keyword boost
+with real **BM25 ranking** from SQLite's built-in FTS5 — improving recall for
+paraphrase and ranking quality as the corpus grows, **without adding a single
+dependency and without giving up the local-only / offline promise.**
+
+Non-goals: embeddings, vector search, entity graphs (see §9).
+
+## 2. Current retrieval, precisely
+
+Two stages, both in-process:
+
+1. **Candidate fetch** — `MemoryStorage.searchByContext()` (`src/memory/storage.ts:666`).
+   When keywords are present it hard-filters rows with `value LIKE ?` per keyword
+   (`storage.ts:698-713`): ≥3 keywords → require ≥2 matches; <3 → match any. The
+   match is against the **raw JSON string of the `value` column** — so it matches
+   field names as well as content, and misses any paraphrase.
+2. **Re-rank** — `MemoryRetrieval.calculateRelevance()` (`src/core/retrieval.ts:141`).
+   A multiplicative pipeline: base `relevance_score` × keyword boost
+   (`retrieval.ts:150-174`, up to ~6× via `String.includes`) × time-decay
+   forgetting curve (`:176-181`) × project/file boosts (`:184-189`) × strength
+   (`:191-193`) × evidence (`:196-198`) × helpfulness prior (`:201-204`) ×
+   staleness penalty (`:206-212`). Then sorted by `TYPE_PRIORITY` then score,
+   sliced to top 5 (`:105-128`).
+
+The weakest link is the **lexical layer** — steps 1 and the boost in step 2.
+Everything else (decay, strength, evidence, project scoping) is worth keeping
+exactly as-is; this change is surgical to the lexical component.
+
+## 3. Proposed design
+
+### 3.1 An external-content FTS5 table, kept in sync by triggers
+
+Add one virtual table plus three triggers. **No change to the `memories` table
+and no change to the TypeScript write path** — the triggers do the syncing in
+SQL, so every writer (upsert, import, janitor, migrations) stays covered
+automatically. That decoupling is the main reason to prefer this over a
+`search_text` column maintained in `save()`.
+
+```sql
+-- external-content FTS mirror of memories.value, keyed by memories.id
+CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+  value,
+  content='memories',
+  content_rowid='id'
+);
+
+CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+  INSERT INTO memories_fts(rowid, value) VALUES (new.id, new.value);
+END;
+CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+  INSERT INTO memories_fts(memories_fts, rowid, value) VALUES('delete', old.id, old.value);
+END;
+CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+  INSERT INTO memories_fts(memories_fts, rowid, value) VALUES('delete', old.id, old.value);
+  INSERT INTO memories_fts(rowid, value) VALUES (new.id, new.value);
+END;
+```
+
+**v1 indexes the raw `value` JSON.** The default `unicode61` tokenizer splits on
+braces/quotes/punctuation, so JSON structure mostly falls away; field-name tokens
+(`what_failed`, `content`) are minor noise and can be dropped later by indexing a
+derived plaintext projection (a refinement, not a v1 requirement).
+
+### 3.2 Candidate fetch via MATCH
+
+Replace the `LIKE` branch in `searchByContext` with an FTS `MATCH` that returns
+candidate ids **and** their BM25 rank, while preserving the existing scope
+predicate (`project_id = ? OR scope = 'universal' OR project_id IS NULL`,
+`storage.ts:680-684`) and type filter:
+
+```sql
+SELECT m.*, bm25(memories_fts) AS bm25_rank
+FROM memories_fts
+JOIN memories m ON m.id = memories_fts.rowid
+WHERE memories_fts MATCH ?
+  AND (m.project_id = ? OR m.scope = 'universal' OR m.project_id IS NULL)
+ORDER BY bm25_rank;      -- SQLite bm25 is negative; more-negative = better
+```
+
+Empty/purely-stopword queries keep today's behavior (return all scoped rows, no
+lexical filter) so `load_rules`-style "give me everything" calls are unaffected.
+
+### 3.3 Query sanitization (must-have, not optional)
+
+Raw user keywords fed to `MATCH` are a **syntax hazard** — bare `AND`/`OR`/`NEAR`,
+hyphens, quotes, and `*` are FTS5 operators and throw `SQLITE_ERROR` on malformed
+input. Every term must be wrapped as a quoted phrase and OR-joined:
+
+```
+"kaggle" OR "submission" OR "api"
+```
+
+A bad query must **fall back to the LIKE path**, never crash retrieval.
+
+### 3.4 Score fusion
+
+Compute a normalized `bm25Score ∈ [0,1]` per candidate (min-max across the
+candidate set, Mem0-style) and **replace** the `includes()` boost at
+`retrieval.ts:150-174` with it — leaving every other multiplicative term
+untouched:
+
+```ts
+// was: score *= 1 + matchRatio * 3.0 (+1.5 all-match / ×0.3 no-overlap)
+score *= 1 + wLexical * bm25Score;   // wLexical ~ 3.0 to preserve current dynamic range
+```
+
+Keeping the fusion multiplicative means decay/strength/evidence/project boosts
+behave identically — the only thing that changes is *how the lexical signal is
+measured*. This bounds the blast radius to one term.
+
+## 4. Migration & backfill
+
+In `migrateSchema()` (`storage.ts`, alongside the existing `CREATE INDEX IF NOT
+EXISTS` migrations ~`:119-149`):
+
+1. Feature-detect FTS5 (see §5). If absent → skip everything, leave a flag off.
+2. Create the vtable + triggers (idempotent).
+3. Backfill once: `INSERT INTO memories_fts(rowid, value) SELECT id, value FROM memories`
+   guarded by "only if the FTS table is empty."
+
+Backfill cost is a single scan — trivial at the 10k-row cap.
+
+## 5. Feature detection, fallback, rollback
+
+- **Detect** at init by attempting `CREATE VIRTUAL TABLE … USING fts5` in a
+  try/catch (verified working in the bundled `better-sqlite3`, but self-built or
+  exotic SQLite may lack it). On failure, set `ftsAvailable = false` and use the
+  existing LIKE path everywhere. **Retrieval must work identically with FTS off.**
+- **Rollback is safe and non-destructive:** the FTS table + triggers are derived,
+  redundant data. `DROP TRIGGER … ; DROP TABLE memories_fts;` reverts to LIKE with
+  zero risk to `memories`.
+
+## 6. Config / opt-in
+
+`CLAUDE_RECALL_RETRIEVAL = fts | like`. Ship **`like` as default** first so the
+change is inert on upgrade, flip to `fts` as default only after the benchmark
+(§7) shows a win. This mirrors how AUTO_DEMOTE / the janitor shipped off-by-default.
+
+## 7. Benchmark tie-in (finding #4)
+
+This change is the reason to stand up the LongMemEval-subset harness **first**:
+without a before/after number we can't tell whether BM25 actually helps or just
+adds surface area. Track both **recall/accuracy** and **tokens-per-query** (the
+2000-token `load_rules` budget is the natural denominator). The harness doubles as
+the regression guard for the fusion weights.
+
+## 8. Risks / open questions
+
+- **Tokenizing raw JSON** dilutes ranking with field-name tokens. Acceptable for
+  v1; the clean fix (derived plaintext column) means a write-path change.
+- **`is_active` / superseded rows**: `searchByContext` currently returns inactive
+  rows too (no `is_active` predicate at `storage.ts:677`). Confirm FTS candidate
+  fetch matches whatever the intended active-set semantics are — don't silently
+  change them in this PR.
+- **Fusion weight `wLexical`** needs tuning against the benchmark; the ×0.3
+  no-overlap penalty has no direct BM25 equivalent (non-matches simply aren't
+  returned by MATCH) — verify that doesn't over-promote weak matches.
+- **WAL + triggers**: triggers run inside the same transaction as the write; the
+  existing `wal_checkpoint(TRUNCATE)` after writes is unaffected.
+
+## 9. Later phase (out of scope here)
+
+Optional local embeddings (`sqlite-vec` or a small local model) for semantic
+similarity, gated behind an opt-in flag so the default stays dependency-free —
+fused as a third signal exactly like Mem0 (semantic + BM25 + entity). BM25 first;
+it captures most of the paraphrase win at none of the cost.
+
+## 10. Suggested PR breakdown
+
+1. FTS vtable + triggers + migration + feature-detect + backfill (no retrieval
+   wiring yet — pure additive, inert).
+2. LongMemEval-subset benchmark harness (measure the `LIKE` baseline).
+3. Candidate fetch via MATCH + sanitization + fusion, behind `CLAUDE_RECALL_RETRIEVAL=fts`.
+4. Flip default to `fts` once the benchmark confirms the win.

package/docs/design-memory-links.md

@@ -0,0 +1,281 @@
+# Design note: `memory_links` — a typed relationship edge for memories
+
+**Status:** proposal — not yet approved for implementation
+**Date:** 2026-08-17 · **Baseline:** v0.37.2
+**Scope of this note:** one additive SQLite table that generalizes the existing
+`superseded_by` edge into typed links between memories, plus an optional one-hop
+expansion in retrieval. **No LLM, no new runtime dependency, no graph DB.**
+Deliberately *not* a mem0-style knowledge graph — see
+`docs/comparison-mem0.md` for why that is over-engineered for this corpus.
+
+---
+
+## 1. Goal
+
+Recall already knows how memories relate to each other — but it throws almost all
+of that structure away. Today the only persisted edge is `memories.superseded_by`
+(one column, overloaded), and several relationships we *compute at write time*
+(fuzzy near-duplicates, failure→fix pairings, promotion provenance) are either
+collapsed into a single value blob or discarded after a boolean/sentinel is set.
+
+The goal is a **single typed-edge table** that:
+- generalizes the one existing edge (`supersedes`) into a small relation
+  vocabulary,
+- **persists structure we already derive** rather than recomputing or losing it,
+- enables a bounded **one-hop expansion** at retrieval ("show the current rule
+  *and* what it replaced"; "this failure *and* its fix"),
+
+all with **zero new dependency**, additive and inert by default, and a
+non-destructive rollback — the same discipline as the FTS5 note.
+
+**Non-goals:** LLM triple extraction, entity-relationship modeling, multi-hop
+traversal, a graph database backend. If a traversal consumer never materializes,
+this table is cheap to drop.
+
+## 2. What relationships exist today, precisely
+
+### 2.1 The one real edge: `superseded_by`
+
+The old row carries the edge: `MemoryStorage.markSuperseded`
+(`src/memory/storage.ts:949-956`) sets `is_active=0, superseded_by=<winning key>,
+superseded_at=<now>`. It is driven from two places:
+- **Preference override** — `MemoryService.storePreferenceWithOverride`
+  (`src/services/memory.ts:407-451`) → `markSupersededPreferences`
+  (`memory.ts:479-497`), joining old→new rows by `preference_key`.
+- **Fuzzy newest-wins / hygiene** — but here `superseded_by` is **overloaded**
+  with *sentinel strings* rather than a key: `'auto-demote'`
+  (`storage.ts:1029`), `'auto-dedup'` (`storage.ts:1198`), `'janitor'`
+  (`storage.ts:1062-1068`), enumerated as `REVIVABLE_SENTINELS`
+  (`storage.ts:316`). `promoteRule` (`storage.ts:1218-1228`) revives only
+  sentinel-superseded rows, refusing rows superseded by a real key.
+
+So `superseded_by` conflates two meanings (a real old→new pointer vs. a
+"demoted-by-process" tag) on one column. That overloading is exactly why a
+*separate* typed-edge table is cleaner than adding more columns here.
+
+### 2.2 Relationships we compute but do not persist as edges
+
+- **Fuzzy near-duplicate** — `findFuzzyDuplicate` (Jaccard ≥ 0.65,
+  `storage.ts:324-353`, used in `save()` at `:459-468`) and retroactive
+  `dedupSimilar` (`storage.ts:1121-1210`). `dedupSimilar` computes
+  `{winnerKey, loserKey, similarity}` per collapse **but returns it only** — the
+  loser is marked `superseded_by='auto-dedup'` and the graded similarity is lost.
+- **Failure → fix pairing** — `tool-outcome-watcher.ts` pairs a later Bash
+  success to a pending failure (Jaccard ≥ 0.3 within a 5-min window,
+  `:276-309`) and **merges the fix into the failure's own value blob**
+  (`mergeIntoValue(pf.memoryKey, { what_should_do: 'Fix: …' })`, `:293-295`).
+  The pairing state lives in an **ephemeral session JSON file**
+  (`<sessionId>-failures.json`, `:49-74`), never a DB edge.
+- **Promotion provenance** — `PromotionEngine.promote`
+  (`src/services/promotion-engine.ts:73-112`) mints a `promoted_<ts>_<rand>`
+  memory and writes its key back to `candidate_lessons.promoted_memory_key`
+  (`updateLessonStatus`, `src/services/outcome-storage.ts:180-185`). This is a
+  soft lesson→memory FK; the promoted memory has **no** column pointing to its
+  source evidence — only `evidence_count`/`source` embedded in its JSON value.
+
+### 2.3 The soft-FK neighborhood a links table would join
+
+`outcome-storage.ts` already maintains several key/id references to
+`memories.key` with no SQL `REFERENCES` constraint (all created in
+`storage.ts:172-280`): `candidate_lessons.promoted_memory_key`,
+`memory_stats.memory_key`, `rule_injection_events.rule_key`, plus
+`*.episode_id → episodes.id`. A `memory_links` table would live beside these and
+follow the same soft-FK convention.
+
+**Content-hash is exact-equality only** (`computeContentHash`,
+`storage.ts:292-305`; checked in `save()` at `:408-434`) — there is no existing
+notion of *graded* similarity surviving anywhere. That is the gap §3.2 fills for
+`refines`/`duplicate-of`.
+
+## 3. Proposed design
+
+### 3.1 The table
+
+One table, alongside the outcome-storage tables, created in the same
+`migrateSchema()` block:
+
+```sql
+CREATE TABLE IF NOT EXISTS memory_links (
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  from_key    TEXT NOT NULL,          -- soft FK -> memories.key
+  to_key      TEXT NOT NULL,          -- soft FK -> memories.key
+  relation    TEXT NOT NULL,          -- see vocabulary below
+  strength    REAL,                   -- e.g. Jaccard for refines/duplicate-of; NULL otherwise
+  source      TEXT NOT NULL,          -- which mechanism created it (override|dedup|fix-pairing|promotion)
+  created_at  INTEGER NOT NULL,
+  UNIQUE(from_key, to_key, relation)
+);
+
+CREATE INDEX IF NOT EXISTS idx_memory_links_from ON memory_links(from_key);
+CREATE INDEX IF NOT EXISTS idx_memory_links_to   ON memory_links(to_key);
+CREATE INDEX IF NOT EXISTS idx_memory_links_rel  ON memory_links(relation);
+```
+
+**Directed edges.** `from_key → to_key`. Read "`from` *relation* `to`":
+`old supersedes-> new` is stored as `from_key=new, to_key=old, relation='supersedes'`
+(the *new* memory supersedes the old), matching the natural query direction
+("given the memory I surfaced, what did it replace?").
+
+**Relation vocabulary (v1, closed set — validate on insert):**
+
+| relation | meaning | source signal (already computed) |
+|---|---|---|
+| `supersedes` | A replaces B (versioning) | preference override / real-key `markSuperseded` (§2.1) |
+| `duplicate-of` | A collapsed B (near-identical) | `dedupSimilar` winner→loser, `strength`=Jaccard (§2.2) |
+| `refines` | A is a fuzzy relative of B, kept | `findFuzzyDuplicate` near-miss below the collapse threshold |
+| `fixed-by` | failure A resolved by B | fix pairing (§2.2) — **needs a target**, see §3.3 |
+| `promoted-from` | lesson memory A promoted from source B | promotion provenance (§2.2) |
+
+`contradicts` and `depends-on` are **deferred** — we have no non-LLM signal for
+them today (see §7). Keep the set closed so retrieval can reason about it.
+
+### 3.2 Populating links — persist, don't recompute
+
+Each link comes from a mechanism that **already runs**; we add a single insert
+next to work already being done. No new scans, no LLM.
+
+- **`supersedes`** — in `markSuperseded` (`storage.ts:949-956`), when
+  `supersededBy` is a **real key** (not a `REVIVABLE_SENTINELS` value), also
+  insert `(from=supersededBy, to=key, relation='supersedes', source='override')`.
+  Sentinel supersessions are skipped — they are process hygiene, not semantic
+  edges. This is the migration of the one existing edge onto the new table.
+- **`duplicate-of`** — in `dedupSimilar` (`storage.ts:1121-1210`), where
+  `{winnerKey, loserKey, similarity}` is *already computed and currently
+  discarded*, insert `(from=winnerKey, to=loserKey, relation='duplicate-of',
+  strength=similarity, source='dedup')` before marking the loser.
+- **`refines`** — in `findFuzzyDuplicate`'s caller in `save()`
+  (`storage.ts:459-468`): when a candidate is a near-match but *above* the
+  keep-threshold (not collapsed), record `refines` with the Jaccard strength
+  instead of throwing the score away.
+- **`promoted-from`** — in `PromotionEngine.promote` (`promotion-engine.ts:110`),
+  right where it already calls `updateLessonStatus(candidateId, 'promoted',
+  key)`: insert `(from=key, to=<best source memory key>, relation='promoted-from',
+  source='promotion')`. Note the source is a *lesson/evidence* reference, so this
+  one may point at a `candidate_lessons.id` rather than a `memories.key`; see §7.
+- **`fixed-by`** — see §3.3 (the only source that needs a shape change first).
+
+All inserts are best-effort and wrapped so a link failure never blocks the
+underlying write — links are an optimization, exactly like the FTS5 mirror.
+
+### 3.3 The `fixed-by` caveat
+
+Today the fix is **merged into the failure's own value blob**
+(`what_should_do`), so there is no second memory to point at — `from=failure,
+to=fix` has no `to`. Two honest options:
+1. **v1: skip `fixed-by`.** The fix already lives in the failure memory; a
+   self-link adds nothing. Lowest effort, no behavior change.
+2. **later: stop merging, store the fix as its own `solution`-type memory** and
+   link `failure --fixed-by--> solution`. This is a capture-path change with real
+   value (the fix becomes independently retrievable) but is out of scope here and
+   should be its own proposal.
+
+Recommend option 1 for v1: `fixed-by` stays in the vocabulary but is not emitted
+until the capture side is reworked.
+
+### 3.4 Consuming links — optional one-hop expansion, off the hot path
+
+Retrieval stays exactly as it is (`src/core/retrieval.ts` — keyword × decay ×
+strength × evidence × …, top-5). Links are consumed **after** ranking, not inside
+the scoring loop, so the hot path is untouched:
+
+```ts
+// after findRelevant() returns the top-5, before formatting for load_rules:
+for (const m of top) {
+  m.related = storage.getLinks(m.key, ['supersedes', 'refines']); // one indexed lookup
+}
+```
+
+The MCP `load_rules` / rule-injector formatter can then optionally append
+"↳ replaces: <old snippet>" under a surfaced rule. This is **presentation-layer
+enrichment**, gated behind a flag (§5), and never changes ranking or the top-5
+selection — bounding the blast radius to the formatter.
+
+## 4. Migration & backfill
+
+In `migrateSchema()` (`storage.ts`, alongside the outcome-storage table
+migrations `:172-280`):
+
+1. Create `memory_links` + indexes (idempotent).
+2. **Backfill `supersedes` once** from existing data:
+   `INSERT OR IGNORE INTO memory_links(from_key, to_key, relation, source, created_at)
+   SELECT superseded_by, key, 'supersedes', 'override', superseded_at
+   FROM memories
+   WHERE superseded_by IS NOT NULL
+     AND superseded_by NOT IN ('auto-demote','auto-dedup','janitor')`
+   — i.e. only real-key supersessions, sentinels excluded. Guard with "only if
+   `memory_links` is empty." Trivial at the 10k-row cap.
+3. No backfill is possible for `duplicate-of`/`refines`/`promoted-from` (the
+   graded signal was never stored) — those accrue going forward.
+
+## 5. Feature detection, config, rollback
+
+- **Opt-in:** `CLAUDE_RECALL_MEMORY_LINKS = off | write | expand`.
+  `off` (default on upgrade) = no inserts, no reads — fully inert.
+  `write` = populate links but don't consume them (lets the table accrue data
+  and be inspected via CLI before trusting it in retrieval).
+  `expand` = also do the §3.4 one-hop enrichment.
+  Ships **`off` by default**, mirroring how the janitor / AUTO_DEMOTE / the
+  proposed FTS5 flag shipped.
+- **Rollback is non-destructive:** the table is derived, redundant data.
+  `DROP TABLE memory_links;` reverts with zero risk to `memories` — the
+  authoritative `superseded_by` column is untouched (we mirror it, never replace
+  it).
+- **Retention:** add `memory_links` to `pruneOldData`
+  (`outcome-storage.ts:391-417`) — delete edges whose `from_key`/`to_key` no
+  longer exist in `memories` (orphan sweep), same pattern as the `memory_stats`
+  orphan cleanup (`:408-410`).
+
+## 6. Why this is worth doing (and where it stops)
+
+- It **captures signal already computed and currently discarded**
+  (`dedupSimilar` similarity, `findFuzzyDuplicate` near-misses) — near-zero
+  marginal cost, since the work already happens.
+- It **de-overloads `superseded_by`** by giving semantic edges a typed home,
+  without a destructive migration.
+- It delivers the concretely useful query — "current rule + what it replaced",
+  "promoted lesson + its provenance" — that mem0's graph is credited for, at a
+  fraction of the cost and with **no LLM and no new dependency**.
+
+It stops short of a knowledge graph on purpose: no entity extraction, no
+multi-hop, no `contradicts`/`depends-on` until a non-LLM signal exists. If the
+one-hop expansion proves unused, the table is dropped and nothing else changes.
+
+## 7. Risks / open questions
+
+- **`superseded_by` overloading** must be respected in the `supersedes` emitter
+  and backfill — never mirror a sentinel value as a semantic edge. The
+  `REVIVABLE_SENTINELS` list (`storage.ts:316`) is the single source of truth for
+  the exclusion.
+- **`promoted-from` target type.** Promotion provenance points at a
+  *candidate_lesson*/evidence, not always a `memories.key`. Either (a) point
+  `to_key` at the source memory when one exists, or (b) give the table a
+  nullable `to_kind` discriminator. Recommend deferring `promoted-from` to a
+  second pass and shipping v1 with just `supersedes` + `duplicate-of` + `refines`.
+- **Edge staleness.** A `to_key` can be pruned/superseded after the link is
+  written; the §5 orphan sweep handles deletion, but a link to a now-*inactive*
+  memory should be filtered at read time (join `is_active` in `getLinks`).
+- **No demand yet.** The rule-injector asks single-hop, relevance-ranked
+  questions; until `load_rules`/injector formatting actually renders related
+  edges, the `expand` mode has no reader. Ship `write` mode first, inspect the
+  accrued edges via a CLI (`claude-recall links <key>`), and only wire `expand`
+  once the data looks useful — same "measure before flipping the default"
+  discipline as the FTS5 benchmark.
+- **Directionality bugs.** `supersedes` is stored new→old; the backfill maps
+  `superseded_by`(=winner) → `from_key` and `key`(=loser) → `to_key`. Get this
+  wrong and expansion shows the *old* text as current. One focused unit test on
+  the backfill direction is mandatory.
+
+## 8. Suggested PR breakdown
+
+1. `memory_links` table + indexes + migration + backfill of real-key
+   `supersedes` + orphan sweep in `pruneOldData`. Pure additive, inert
+   (`MEMORY_LINKS=off`). Includes the backfill-direction unit test.
+2. Emit `supersedes` (real-key branch of `markSuperseded`) and `duplicate-of` /
+   `refines` (from `dedupSimilar` / `findFuzzyDuplicate`), behind
+   `MEMORY_LINKS=write`. + `getLinks()` accessor and a `claude-recall links <key>`
+   CLI for inspection.
+3. One-hop `expand` enrichment in the `load_rules` / rule-injector formatter,
+   behind `MEMORY_LINKS=expand`. Presentation only; no ranking change.
+4. (separate proposal) Rework failure→fix capture to store the fix as its own
+   `solution` memory and emit `fixed-by`; add `promoted-from` with a resolved
+   target. Only after 1–3 prove the model useful.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-recall",
-  "version": "0.37.2",
+  "version": "0.38.0",
   "description": "Persistent memory for Claude Code and Pi with native Skills integration, automatic capture, failure learning, and project scoping",
   "main": "dist/index.js",
   "bin": {
@@ -41,6 +41,7 @@
     "test:coverage:unit": "jest --coverage --maxWorkers=50% tests/unit",
     "test:coverage:integration": "jest --coverage --maxWorkers=50% tests/integration",
     "test:benchmarks": "jest tests/benchmarks",
+    "bench:retrieval": "ts-node tests/benchmarks/retrieval-benchmark.ts",
     "test:experimental": "jest tests/experimental",
     "build": "tsc && mkdir -p dist/memory dist/config && cp src/memory/schema.sql dist/memory/ && (cp -r src/config/* dist/config/ 2>/dev/null || true) && chmod +x dist/cli/claude-recall-cli.js",
     "build:cli": "tsc && chmod +x dist/cli/claude-recall-cli.js",