@nxuss/lemma 1.16.0 → 1.17.0

package/dist/esm/subconscious/TheBrainV2.js

@@ -13,10 +13,25 @@ import fs from 'fs';
 import path from 'path';
 import os from 'os';
 import crypto from 'crypto';
+import { execSync } from 'child_process';
 import { getSymbolSurgicalContext, normalizeForFreshness } from '../utils/SymbolSurgicalContext.js';
+import { EmbeddingSidecar, embeddingsEnabled, semanticRerank } from './BrainEmbeddings.js';
 // ─── BM25 Constants ───────────────────────────────────────────────────────────
 const BM25_K1 = 1.5; // Term saturation (1.2-2.0)
 const BM25_B = 0.75; // Length normalization (0-1)
+// ─── Recency decay ────────────────────────────────────────────────────────────
+//
+// Deliberately gentle and grace-periodded. Age is a weak proxy for "no longer true" —
+// plenty of architectural decisions stay correct for years — so this is sized to break ties
+// between comparable memories, not to bury an old one that is still the best match. A
+// memory that goes stale is caught by hashing, which is the real signal; this only covers
+// the case hashing can't see, where a memory tracks no files at all.
+/** Days a memory is treated as current with no penalty at all. */
+const RECENCY_GRACE_DAYS = 30;
+/** Days past the grace period at which the penalty reaches its cap. */
+const RECENCY_FULL_DECAY_DAYS = 365;
+/** Cap, in the same 0-1 similarity space as the popularity/outcome/demerit adjustments. */
+const MAX_RECENCY_PENALTY = 0.10;
 // ─── Storage Paths ────────────────────────────────────────────────────────────
 /**
  * LEMMA_BRAIN_DIR redirects storage — required so tests never write into the user's real
@@ -27,6 +42,273 @@ const BRAIN_DIR = process.env.LEMMA_BRAIN_DIR || path.join(os.homedir(), '.lemma
 const ENTRIES_FILE = path.join(BRAIN_DIR, 'entries.ndjson');
 const INDEX_FILE = path.join(BRAIN_DIR, 'index.json');
 const META_FILE = path.join(BRAIN_DIR, 'meta.json');
+/** Advisory cross-process write lock. See acquireLock() for why it is advisory, not a guarantee. */
+const LOCK_FILE = path.join(BRAIN_DIR, '.write.lock');
+/** Sidecar vector cache for the optional semantic re-rank. See BrainEmbeddings.ts. */
+const EMBEDDINGS_FILE = path.join(BRAIN_DIR, 'embeddings.ndjson');
+// ─── Cross-process durability ─────────────────────────────────────────────────
+//
+// Every MCP client session starts its own server process, and each one holds the whole
+// corpus in memory. Before this section existed, save() blind-rewrote entries.ndjson in
+// full: two concurrent sessions meant whichever flushed last silently erased everything the
+// other had stored since it loaded. That is the exact usage pattern the tool descriptions
+// advertise ("memories from ALL your projects"), so it was not a rare race.
+//
+// Three things fix it, and none of them require a database:
+//   1. save() merges what is on disk into memory before writing, so a full rewrite can only
+//      ever be a superset — a writer can add, never subtract (except via tombstones).
+//   2. Writes go through a temp file + rename, so a crash mid-write can't truncate the
+//      corpus into something load() would discard as "corrupt, start fresh".
+//   3. An advisory lock file serializes the read-merge-write window between processes.
+// Reads get the same benefit through syncIfChanged(): a long-lived server picks up a
+// sibling session's memories on the next search instead of at the next restart.
+/** A lock older than this is assumed to belong to a process that died holding it. */
+const LOCK_STALE_MS = 10000;
+/** How long a writer waits for another process's lock before proceeding regardless. */
+const LOCK_WAIT_MS = 2000;
+/** Deleted ids stop being suppressed after this long — a tombstone list must not grow forever. */
+const TOMBSTONE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+/** Blocking sleep. save() is synchronous and its callers must never observe a partial corpus. */
+function sleepSync(ms) {
+    try {
+        Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+    }
+    catch {
+        // SharedArrayBuffer unavailable (locked-down runtime) — spin instead of failing.
+        const until = Date.now() + ms;
+        while (Date.now() < until) { /* busy wait */ }
+    }
+}
+/**
+ * Take the write lock, or give up after LOCK_WAIT_MS and return null.
+ *
+ * Deliberately advisory and fail-open: a caller that can't get the lock still writes. The
+ * merge in save() is what actually prevents data loss; the lock only narrows the window
+ * where two mergers interleave. Blocking a memory write forever on a lock — or throwing —
+ * would be a worse outcome than the rare interleave it prevents.
+ */
+function acquireLock() {
+    const deadline = Date.now() + LOCK_WAIT_MS;
+    for (;;) {
+        try {
+            return fs.openSync(LOCK_FILE, 'wx');
+        }
+        catch {
+            try {
+                if (Date.now() - fs.statSync(LOCK_FILE).mtimeMs > LOCK_STALE_MS) {
+                    fs.unlinkSync(LOCK_FILE);
+                    continue; // retry immediately against the freed lock
+                }
+            }
+            catch {
+                continue; // lock vanished between open and stat — the holder just released it
+            }
+            if (Date.now() >= deadline)
+                return null;
+            sleepSync(15);
+        }
+    }
+}
+function releaseLock(fd) {
+    if (fd === null)
+        return;
+    try {
+        fs.closeSync(fd);
+    }
+    catch { /* already closed */ }
+    try {
+        fs.unlinkSync(LOCK_FILE);
+    }
+    catch { /* already gone */ }
+}
+/**
+ * Write via temp file + rename. rename(2) is atomic within a filesystem, so a reader either
+ * sees the whole previous file or the whole new one — never a truncated prefix, which
+ * load()'s catch-all would have silently turned into an empty Brain.
+ */
+function writeFileAtomic(target, data) {
+    const tmp = `${target}.tmp.${process.pid}.${Date.now().toString(36)}`;
+    try {
+        fs.writeFileSync(tmp, data, 'utf8');
+        fs.renameSync(tmp, target);
+    }
+    catch (err) {
+        try {
+            fs.unlinkSync(tmp);
+        }
+        catch { /* nothing to clean up */ }
+        throw err;
+    }
+}
+function stampOf(file) {
+    try {
+        const s = fs.statSync(file);
+        return { mtimeMs: s.mtimeMs, size: s.size };
+    }
+    catch {
+        return null;
+    }
+}
+function sameStamp(a, b) {
+    if (a === null || b === null)
+        return a === b;
+    return a.mtimeMs === b.mtimeMs && a.size === b.size;
+}
+/** Parses entries.ndjson exactly the way load() does, without touching instance state. */
+function readEntriesFile() {
+    const out = new Map();
+    let raw;
+    try {
+        raw = fs.readFileSync(ENTRIES_FILE, 'utf8');
+    }
+    catch {
+        return out;
+    }
+    for (const line of raw.split('\n')) {
+        if (!line)
+            continue;
+        try {
+            const entry = JSON.parse(line);
+            if (!entry || typeof entry.id !== 'string')
+                continue;
+            // Entries written before the format was compacted still carry `terms`; newer ones
+            // don't. Rebuilding from termFreq covers both without a migration step.
+            if (!Array.isArray(entry.terms))
+                entry.terms = Object.keys(entry.termFreq || {});
+            out.set(entry.id, entry);
+        }
+        catch { /* skip corrupt lines */ }
+    }
+    return out;
+}
+/** Reads the tombstone map straight off disk, for absorbing a peer process's deletions. */
+function readTombstonesFile() {
+    try {
+        return readTombstones(JSON.parse(fs.readFileSync(META_FILE, 'utf8')));
+    }
+    catch {
+        return new Map();
+    }
+}
+/** Reads the tombstone map out of a parsed meta.json, tolerating older files without one. */
+function readTombstones(meta) {
+    const out = new Map();
+    const raw = meta?.tombstones;
+    if (!raw || typeof raw !== 'object')
+        return out;
+    for (const [id, at] of Object.entries(raw)) {
+        if (typeof at === 'string')
+            out.set(id, at);
+    }
+    return out;
+}
+/** Effective age anchor: a refreshed memory is as young as its last re-verification. */
+function effectiveTime(entry) {
+    const t = Date.parse(entry.refreshedAt || entry.timestamp || '');
+    return Number.isFinite(t) ? t : 0;
+}
+/**
+ * Union two views of the corpus. Never subtracts: an id present in either side survives
+ * unless it is tombstoned, because "absent from my copy" and "deleted" are indistinguishable
+ * from inside one process, and guessing wrong loses a user's memory permanently.
+ *
+ * For an id on both sides, the newer version (by refreshedAt/timestamp) supplies the content
+ * and the counters take the max of both — `hits` and `demerits` are monotonic tallies that
+ * each process accumulated independently, so max is the only merge that doesn't discard
+ * feedback one of them collected.
+ */
+export function mergeEntryMaps(mine, theirs, tombstones = new Set()) {
+    const merged = new Map();
+    const put = (entry) => {
+        if (tombstones.has(entry.id))
+            return;
+        const existing = merged.get(entry.id);
+        if (!existing) {
+            merged.set(entry.id, entry);
+            return;
+        }
+        const winner = effectiveTime(entry) > effectiveTime(existing) ? entry : existing;
+        merged.set(entry.id, {
+            ...winner,
+            hits: Math.max(entry.hits || 0, existing.hits || 0),
+            demerits: Math.max(entry.demerits || 0, existing.demerits || 0),
+        });
+    };
+    for (const entry of mine.values())
+        put(entry);
+    for (const entry of theirs.values())
+        put(entry);
+    return merged;
+}
+// ─── Git context ──────────────────────────────────────────────────────────────
+/**
+ * Current HEAD and branch, cached briefly.
+ *
+ * Read at store time and again when reporting a stale result, so the Brain can tell
+ * "someone edited this function" apart from "you switched branches" — a content hash
+ * reports both identically, and only the first is a reason to distrust the memory.
+ *
+ * Cached because a stale result is attributed inside the search result loop: without it,
+ * one search over five stale hits would spawn ten git processes.
+ */
+const GIT_CONTEXT_TTL_MS = 5000;
+let _gitContextCache = null;
+export function currentGitContext(cwd = process.cwd()) {
+    const now = Date.now();
+    if (_gitContextCache && _gitContextCache.cwd === cwd && now - _gitContextCache.at < GIT_CONTEXT_TTL_MS) {
+        return _gitContextCache.ctx;
+    }
+    let ctx = {};
+    try {
+        // Not a git repo, no git binary, detached worktree — all of these must produce {} and
+        // never throw. Git context is an explanatory nicety; nothing depends on it existing.
+        const run = (args) => execSync(`git ${args}`, { cwd, encoding: 'utf8', timeout: 1500, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+        const commit = run('rev-parse HEAD');
+        const branch = run('rev-parse --abbrev-ref HEAD');
+        ctx = {
+            ...(commit ? { commit } : {}),
+            // "HEAD" is what git reports in a detached checkout — not a branch name, so don't
+            // record it as one and later claim the user "switched branches" away from it.
+            ...(branch && branch !== 'HEAD' ? { branch } : {}),
+        };
+    }
+    catch {
+        ctx = {};
+    }
+    _gitContextCache = { cwd, at: now, ctx };
+    return ctx;
+}
+/** Test seam: forget the cached git context so a test can change branches mid-run. */
+export function resetGitContextCache() {
+    _gitContextCache = null;
+}
+/**
+ * Why a stale entry is stale. Pure attribution over data already gathered — it never
+ * changes whether something is stale, only how the staleness is explained.
+ */
+export function attributeStaleness(entry, git) {
+    if (entry.gitBranch && git.branch && entry.gitBranch !== git.branch) {
+        return { staleCause: 'branch-changed', storedOnBranch: entry.gitBranch };
+    }
+    return { staleCause: 'content-changed' };
+}
+/**
+ * Normalized provenance for an entry, back-deriving one for entries written before `source`
+ * existed. Those overloaded the free-text `provider` field with the same information, so
+ * reading it here keeps historical stats meaningful instead of a wall of "unknown".
+ */
+export function entrySource(entry) {
+    if (entry.source)
+        return entry.source;
+    switch (entry.provider) {
+        case 'test_oracle_auto': return 'auto-test';
+        case 'checkpoint': return 'auto-checkpoint';
+        case 'git-commit':
+        case 'changelog': return 'auto-git';
+        case 'pr-review': return 'auto-pr-review';
+        default: return 'unknown';
+    }
+}
 // ─── Project Scoping ──────────────────────────────────────────────────────────
 /**
  * Stable identity for the project a memory belongs to.
@@ -617,6 +899,11 @@ export class TheBrainV2 {
         this.sessionMisses = 0;
         this.dirty = false;
         this.flushTimer = null;
+        /** State of entries.ndjson as of our last read/write — see syncIfChanged(). */
+        this.diskStamp = null;
+        /** id -> ISO deletion time, for ids that must not come back through a merge. */
+        this.tombstones = new Map();
+        this.sidecar = null;
         this.ensureDir();
         this.load();
     }
@@ -627,36 +914,23 @@ export class TheBrainV2 {
     }
     load() {
         try {
-            // Load entries from NDJSON
-            if (fs.existsSync(ENTRIES_FILE)) {
-                const lines = fs.readFileSync(ENTRIES_FILE, 'utf8').split('\n').filter(Boolean);
-                for (const line of lines) {
-                    try {
-                        const entry = JSON.parse(line);
-                        // Entries written before the format was compacted still carry `terms`; newer
-                        // ones don't. Rebuilding from termFreq covers both without a migration step.
-                        if (!Array.isArray(entry.terms))
-                            entry.terms = Object.keys(entry.termFreq || {});
-                        this.entries.set(entry.id, entry);
-                    }
-                    catch { /* skip corrupt lines */ }
-                }
-            }
-            // Load inverted index
-            if (fs.existsSync(INDEX_FILE)) {
-                const raw = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
-                for (const [term, ids] of Object.entries(raw)) {
-                    this.invertedIndex.set(term, new Set(ids));
-                }
-            }
-            // Load bloom filter and meta
+            // Meta first: the tombstone list decides which entries are allowed back in.
             if (fs.existsSync(META_FILE)) {
                 const meta = JSON.parse(fs.readFileSync(META_FILE, 'utf8'));
-                if (meta.bloom) {
+                if (meta.bloom)
                     this.bloom = new BloomFilter(meta.bloom);
-                }
                 this.avgDocLength = meta.avgDocLength || 0;
+                this.tombstones = readTombstones(meta);
             }
+            this.entries = readEntriesFile();
+            for (const id of this.tombstones.keys())
+                this.entries.delete(id);
+            this.diskStamp = stampOf(ENTRIES_FILE);
+            // The inverted index is rebuilt from the entries rather than read back from
+            // index.json. It is a pure derivation of the entries, so reconstructing it is the
+            // only way it can never disagree with them — and a stale or half-written index.json
+            // used to be able to hide entries from search entirely while they sat on disk intact.
+            this.rebuildIndex();
             this.recalcAvgDocLength();
         }
         catch (err) {
@@ -664,6 +938,52 @@ export class TheBrainV2 {
             this.entries.clear();
             this.invertedIndex.clear();
             this.bloom = new BloomFilter();
+            this.tombstones = new Map();
+        }
+    }
+    /** Recomputes the inverted index from scratch over the current entries. */
+    rebuildIndex() {
+        this.invertedIndex = new Map();
+        for (const entry of this.entries.values()) {
+            for (const term of entry.terms) {
+                let ids = this.invertedIndex.get(term);
+                if (!ids) {
+                    ids = new Set();
+                    this.invertedIndex.set(term, ids);
+                }
+                ids.add(entry.id);
+            }
+        }
+    }
+    /**
+     * Pull in anything another process wrote since we last touched the corpus.
+     *
+     * Called at the top of every read path. Two statSync calls when nothing changed (the
+     * overwhelmingly common case) is far below the cost of the search that follows, and it
+     * turns sibling sessions from a data-loss hazard into a live shared corpus: a memory
+     * stored in one project's session is searchable from another within one tool call.
+     *
+     * Merges rather than reloads, so memories stored locally but not yet flushed survive.
+     */
+    syncIfChanged() {
+        const stamp = stampOf(ENTRIES_FILE);
+        if (sameStamp(stamp, this.diskStamp))
+            return;
+        try {
+            this.absorbPeerTombstones();
+            const onDisk = readEntriesFile();
+            const before = this.entries.size;
+            this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
+            this.diskStamp = stamp;
+            if (this.entries.size !== before || onDisk.size > 0) {
+                this.rebuildIndex();
+                this.recalcAvgDocLength();
+            }
+        }
+        catch {
+            // A read failure here must never break a search: worst case we keep serving the
+            // in-memory corpus, which is exactly the pre-existing behavior.
+            this.diskStamp = stamp;
         }
     }
     recalcAvgDocLength() {
@@ -685,38 +1005,111 @@ export class TheBrainV2 {
         }, 500);
     }
     save() {
+        const fd = acquireLock();
         try {
             this.ensureDir();
+            // Merge before writing. A full rewrite of what this process happens to hold would
+            // erase every memory a sibling session stored since we loaded — the concurrency bug
+            // this whole section exists to close. After the merge the file we write is a superset
+            // of both views, so a writer can only ever add.
+            this.absorbPeerTombstones();
+            const onDisk = readEntriesFile();
+            if (onDisk.size > 0) {
+                this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
+                this.rebuildIndex();
+                this.recalcAvgDocLength();
+            }
+            this.pruneTombstones();
+            // Post-merge, so the file we write respects the cap even when the merge pulled in
+            // entries a peer had already evicted.
+            this.evictIfOverCapacity();
             // Write NDJSON entries. `terms` is dropped: it is exactly Object.keys(termFreq),
             // and persisting both made the entry file 38% redundant bytes that every session
             // re-read at startup. load() reconstructs it.
             const ndjson = Array.from(this.entries.values())
                 .map(({ terms: _terms, ...persisted }) => JSON.stringify(persisted))
                 .join('\n');
-            fs.writeFileSync(ENTRIES_FILE, ndjson, 'utf8');
-            // Write inverted index
+            writeFileAtomic(ENTRIES_FILE, ndjson);
+            // Write inverted index. load() rebuilds this from the entries rather than reading it
+            // back, so it is now purely an inspection artifact for the dashboard and for anyone
+            // poking at the brain directory — kept because removing a file other tooling may read
+            // is not worth the handful of bytes it saves.
             const indexObj = {};
             for (const [term, ids] of this.invertedIndex) {
                 indexObj[term] = Array.from(ids);
             }
-            fs.writeFileSync(INDEX_FILE, JSON.stringify(indexObj), 'utf8');
+            writeFileAtomic(INDEX_FILE, JSON.stringify(indexObj));
             // Write meta
-            fs.writeFileSync(META_FILE, JSON.stringify({
+            writeFileAtomic(META_FILE, JSON.stringify({
                 bloom: this.bloom.serialize(),
                 avgDocLength: this.avgDocLength,
                 totalEntries: this.entries.size,
                 savedAt: new Date().toISOString(),
-            }), 'utf8');
+                tombstones: Object.fromEntries(this.tombstones),
+            }));
+            this.diskStamp = stampOf(ENTRIES_FILE);
             this.dirty = false;
         }
         catch { /* fail silently */ }
+        finally {
+            releaseLock(fd);
+        }
+    }
+    /**
+     * Pull in deletions made by other processes.
+     *
+     * Tombstones are shared state, not per-process bookkeeping: a peer that deleted a memory
+     * wrote the tombstone to meta.json, and a process that still holds the entry in memory
+     * would otherwise merge it right back on its next flush — undoing a deliberate deletion
+     * from a session that had nothing to do with it.
+     */
+    absorbPeerTombstones() {
+        for (const [id, at] of readTombstonesFile()) {
+            if (!this.tombstones.has(id))
+                this.tombstones.set(id, at);
+            const entry = this.entries.get(id);
+            if (entry)
+                this.dropEntry(entry);
+        }
+    }
+    /**
+     * Records an id as deliberately deleted, so a merge with a peer that still holds it
+     * doesn't resurrect it. Without this, forget() would be undone the moment any other
+     * session flushed, and eviction would thrash forever between two processes.
+     */
+    tombstone(id) {
+        this.tombstones.set(id, new Date().toISOString());
+    }
+    /** Drops tombstones older than TOMBSTONE_TTL_MS — by then no peer still holds the entry. */
+    pruneTombstones() {
+        const cutoff = Date.now() - TOMBSTONE_TTL_MS;
+        for (const [id, at] of this.tombstones) {
+            const t = Date.parse(at);
+            if (!Number.isFinite(t) || t < cutoff)
+                this.tombstones.delete(id);
+        }
+    }
+    /** Removes an entry from the in-memory corpus and the inverted index. No persistence. */
+    dropEntry(entry) {
+        this.entries.delete(entry.id);
+        for (const term of entry.terms) {
+            const ids = this.invertedIndex.get(term);
+            if (!ids)
+                continue;
+            ids.delete(entry.id);
+            if (ids.size === 0)
+                this.invertedIndex.delete(term);
+        }
     }
     // ─── Store ────────────────────────────────────────────────────────────────
     /**
      * Store a query+response pair in the brain.
      * Returns false if detected as duplicate (>= dupThreshold similarity).
      */
-    store(query, response, provider = 'generic', dupThreshold = 0.92, filePaths, projectId = deriveProjectId(), outcome, symbolRefs, claimInputs, domain, derivedFrom) {
+    store(query, response, provider = 'generic', dupThreshold = 0.92, filePaths, projectId = deriveProjectId(), outcome, symbolRefs, claimInputs, domain, derivedFrom, options = {}) {
+        // Another session may have stored this exact thing since we loaded; without the sync
+        // the dedup check below would miss it and write a second copy.
+        this.syncIfChanged();
         // Quick bloom check
         const queryKey = query.trim().toLowerCase().substring(0, 200);
         if (this.bloom.has(queryKey)) {
@@ -754,11 +1147,20 @@ export class TheBrainV2 {
         }
         const terms = tokenize(query + ' ' + response);
         const termFreq = termFrequencies(terms);
+        // Salted with random bytes, not just the clock: the id used to be sha1(queryKey + now),
+        // so two memories sharing a query prefix and landing in the same millisecond produced
+        // the same id and the second silently overwrote the first — a lost memory that nothing
+        // reported. Batch paths (git ingest, importBundle, auto-capture) hit that window.
         const id = crypto
             .createHash('sha1')
-            .update(queryKey + Date.now())
+            .update(queryKey + Date.now() + crypto.randomBytes(8).toString('hex'))
             .digest('hex')
             .substring(0, 12);
+        // Recorded so a later stale result can say whether the code changed or the caller
+        // simply moved branches. Never consulted by freshness itself.
+        const git = options.skipGitContext
+            ? { commit: options.gitCommit, branch: options.gitBranch }
+            : currentGitContext();
         const entry = {
             id,
             query: query.trim(),
@@ -766,7 +1168,7 @@ export class TheBrainV2 {
             terms: [...new Set(terms)],
             termFreq,
             provider,
-            timestamp: new Date().toISOString(),
+            timestamp: options.timestamp || new Date().toISOString(),
             hits: 0,
             charCount: query.length + response.length,
             projectId,
@@ -777,6 +1179,9 @@ export class TheBrainV2 {
             claims: claimInputs && claimInputs.length > 0 ? buildClaims(claimInputs) : undefined,
             domain,
             derivedFrom: derivedFrom && derivedFrom.length > 0 ? derivedFrom : undefined,
+            source: options.source || 'manual',
+            ...(git.commit ? { gitCommit: git.commit } : {}),
+            ...(git.branch ? { gitBranch: git.branch } : {}),
         };
         this.entries.set(id, entry);
         // Update inverted index
@@ -791,7 +1196,7 @@ export class TheBrainV2 {
         this.recalcAvgDocLength();
         this.evictIfOverCapacity();
         this.scheduleSave();
-        return { stored: true, reason: 'Stored successfully', conflicts };
+        return { stored: true, reason: 'Stored successfully', id, conflicts };
     }
     /**
      * Evict the lowest-value entries once the Brain is over capacity. Value = hits (proven
@@ -818,15 +1223,11 @@ export class TheBrainV2 {
         });
         const toEvict = ranked.slice(0, TheBrainV2.EVICT_BATCH);
         for (const entry of toEvict) {
-            this.entries.delete(entry.id);
-            for (const term of entry.terms) {
-                const ids = this.invertedIndex.get(term);
-                if (!ids)
-                    continue;
-                ids.delete(entry.id);
-                if (ids.size === 0)
-                    this.invertedIndex.delete(term);
-            }
+            this.dropEntry(entry);
+            // Tombstoned, not just dropped: another session's merge would otherwise hand back
+            // every entry this pass just decided was worthless, and the two processes would
+            // evict-and-resurrect the same 500 entries against each other indefinitely.
+            this.tombstone(entry.id);
         }
         this.recalcAvgDocLength();
     }
@@ -838,6 +1239,8 @@ export class TheBrainV2 {
     search(query, limit = 5, minSimilarity = 0, options = {}) {
         // Internal callers (dedup) must not move the counters the savings ledger reports.
         const countStats = options.countStats !== false;
+        // Pick up anything a sibling session stored since the last read. Two statSync calls.
+        this.syncIfChanged();
         if (this.entries.size === 0) {
             if (countStats)
                 this.sessionMisses++;
@@ -928,6 +1331,16 @@ export class TheBrainV2 {
             // popularity prior above rather than just canceling it out.
             if (s.entry.demerits)
                 similarity -= Math.min(s.entry.demerits * 0.06, 0.25);
+            // Age decay. Every other signal here is about the memory's track record; none of them
+            // notice that the codebase it describes has been rewritten twice since. A confirmed
+            // memory from a year ago is not as likely to be current as yesterday's, and until now
+            // they ranked identically. Kept in the same small, additive register as the rest — it
+            // demotes among relevant candidates, it can't bury one. `refreshedAt` resets the clock,
+            // which is the point of refresh(): a re-verified memory really is current again.
+            const ageDays = (Date.now() - effectiveTime(s.entry)) / 86400000;
+            if (Number.isFinite(ageDays) && ageDays > RECENCY_GRACE_DAYS) {
+                similarity -= Math.min((ageDays - RECENCY_GRACE_DAYS) / RECENCY_FULL_DECAY_DAYS, 1) * MAX_RECENCY_PENALTY;
+            }
             return { entry: s.entry, similarity: Math.max(0, similarity) };
         });
         // 6. Sort and filter
@@ -935,6 +1348,9 @@ export class TheBrainV2 {
         const claimIndex = buildClaimIndex(this.entries);
         const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
         const semanticDiff = options.semanticDiff === true;
+        // Resolved once per search, not per stale hit — five stale results would otherwise
+        // spawn ten git processes for one query.
+        const git = currentGitContext();
         const results = combined
             .filter(r => r.similarity >= minSimilarity)
             .slice(0, limit)
@@ -949,7 +1365,7 @@ export class TheBrainV2 {
                 provider: r.entry.provider,
                 timestamp: r.entry.timestamp,
                 fresh,
-                ...(fresh ? {} : { staleFiles }),
+                ...(fresh ? {} : { staleFiles, ...attributeStaleness(r.entry, git) }),
                 ...(r.entry.outcome ? { outcome: r.entry.outcome } : {}),
                 ...(claims && claims.length > 0 ? { claims } : {}),
                 ...(r.entry.domain ? { domain: r.entry.domain } : {}),
@@ -977,6 +1393,69 @@ export class TheBrainV2 {
      * Entries belonging to a project, including the unscoped ones written before entries
      * carried a projectId — same fallback rule search() uses, so counts and results agree.
      */
+    /**
+     * search(), with an optional semantic re-rank layered on top.
+     *
+     * BM25 still does the retrieving — this only reorders what it already found, and only
+     * when LEMMA_BRAIN_EMBEDDINGS is set and Ollama answers. With the flag off (the default)
+     * this is exactly `search()` plus one boolean check, so callers can use it unconditionally
+     * and no session pays for a feature it hasn't turned on. See BrainEmbeddings.ts.
+     *
+     * The candidate pool is widened before re-ranking: re-ordering the same `limit` results
+     * BM25 already picked can only shuffle them, never surface the memory BM25 ranked 9th
+     * because it happened to use different words — which is the entire point.
+     */
+    async searchHybrid(query, limit = 5, minSimilarity = 0, options = {}) {
+        if (!embeddingsEnabled())
+            return this.search(query, limit, minSimilarity, options);
+        const poolSize = Math.max(limit * 4, 20);
+        // minSimilarity 0 for the pool: a candidate the re-rank would promote must not be cut
+        // by a lexical floor before the semantic score is ever computed. The floor is applied
+        // again below, against the blended score.
+        const pool = this.search(query, poolSize, 0, { ...options, countStats: false });
+        if (pool.length === 0) {
+            if (options.countStats !== false)
+                this.sessionMisses++;
+            return [];
+        }
+        let ranked = pool;
+        try {
+            const scores = await semanticRerank(query, pool.map((r) => ({ id: r.id, similarity: r.similarity, text: `${r.query}\n${r.response}` })), this.embeddingSidecar());
+            if (scores) {
+                ranked = pool
+                    .map((r) => ({ ...r, similarity: scores.get(r.id) ?? r.similarity }))
+                    .sort((a, b) => b.similarity - a.similarity);
+                this.sidecar?.flush(new Set(this.entries.keys()));
+            }
+        }
+        catch {
+            // Re-ranking is an enhancement; a failure in it must never cost the caller the
+            // results BM25 already produced.
+            ranked = pool;
+        }
+        const results = ranked.filter((r) => r.similarity >= minSimilarity).slice(0, limit);
+        // Stats bookkeeping the pool search was told to skip, applied once against the final list
+        // so a hybrid search counts exactly like a lexical one.
+        if (options.countStats !== false) {
+            if (results.length > 0) {
+                this.sessionHits++;
+                const topEntry = this.entries.get(results[0].id);
+                if (topEntry) {
+                    topEntry.hits++;
+                    this.scheduleSave();
+                }
+            }
+            else {
+                this.sessionMisses++;
+            }
+        }
+        return results;
+    }
+    embeddingSidecar() {
+        if (!this.sidecar)
+            this.sidecar = new EmbeddingSidecar(EMBEDDINGS_FILE);
+        return this.sidecar;
+    }
     getEntriesForProject(projectId) {
         return Array.from(this.entries.values()).filter((e) => e.projectId === undefined || e.projectId === projectId);
     }
@@ -1017,6 +1496,7 @@ export class TheBrainV2 {
      * differently-phrased query tomorrow.
      */
     downvote(id) {
+        this.syncIfChanged();
         const entry = this.entries.get(id);
         if (!entry)
             return { ok: false, message: `No entry with id "${id}" in the Brain.` };
@@ -1024,6 +1504,136 @@ export class TheBrainV2 {
         this.scheduleSave();
         return { ok: true, message: `Recorded negative feedback on entry "${id}" (demerits: ${entry.demerits}). It will rank lower and be evicted sooner.` };
     }
+    // ─── Deletion ───────────────────────────────────────────────────────────────
+    /**
+     * Permanently remove one memory.
+     *
+     * The Brain had no way to delete anything: downvote() only demotes, and eviction only
+     * fires at capacity. A memory that is simply wrong, or that captured something that
+     * should never have been stored, had no exit — the only recourse was to downvote it
+     * repeatedly and wait for the corpus to fill up. This is that exit.
+     *
+     * The id is tombstoned as well as dropped, so a concurrent session's merge can't hand it
+     * straight back. The bloom filter cannot un-add a key, so the deleted query may still
+     * register as a possible duplicate later; that costs one extra dedup search and never
+     * produces a wrong answer, which is the right side of that trade for a probabilistic
+     * filter that is rebuilt on the next clear().
+     */
+    forget(id) {
+        this.syncIfChanged();
+        const entry = this.entries.get(id);
+        if (!entry)
+            return { ok: false, message: `No entry with id "${id}" in the Brain.` };
+        this.dropEntry(entry);
+        this.tombstone(id);
+        this.recalcAvgDocLength();
+        this.scheduleSave();
+        return {
+            ok: true,
+            message: `Deleted memory "${id}". It will not come back from another session's copy.`,
+            // Echoed back so a deletion is auditable: the caller (and the user reading the
+            // transcript) can see exactly what was destroyed, since nothing else can recover it.
+            forgotten: { id, query: entry.query, timestamp: entry.timestamp, hits: entry.hits },
+        };
+    }
+    // ─── Re-anchoring ───────────────────────────────────────────────────────────
+    /**
+     * Re-verify a memory against the code as it stands now, keeping its identity.
+     *
+     * Staleness used to be terminal: once tracked evidence changed, an entry was stale
+     * forever, even in the very common case where the insight is still true and the code just
+     * moved. The only workaround was to store a near-duplicate — which store()'s own dedup
+     * guard would often refuse — losing the entry's id, its hit count, its demerits, and every
+     * `derivedFrom` edge pointing at it.
+     *
+     * With no arguments this re-hashes whatever the entry already tracks (and each claim's own
+     * evidence), which is the "yes, I checked, this is still correct" path. Passing filePaths
+     * or symbols instead re-points the entry at new evidence, which is the "the code moved"
+     * path. Either way the caller is asserting the memory is currently true — this tool
+     * records that assertion, it cannot verify it, so it is never called automatically.
+     *
+     * `derivedFrom` dependencies are deliberately not re-anchored: a conclusion inherited from
+     * a memory that is itself stale is exactly what Fase B exists to catch, and silently
+     * clearing that would defeat it. Those come back in `stillStale` instead.
+     */
+    refresh(id, opts = {}) {
+        this.syncIfChanged();
+        const entry = this.entries.get(id);
+        if (!entry)
+            return { ok: false, message: `No entry with id "${id}" in the Brain.` };
+        const retracked = [];
+        const rel = (abs) => {
+            const r = path.relative(process.cwd(), abs);
+            return r.startsWith('..') ? abs : r;
+        };
+        const hasNewTracking = (opts.filePaths?.length || 0) > 0 || (opts.symbols?.length || 0) > 0;
+        if (hasNewTracking) {
+            // Re-point: the new tracking replaces the old wholesale rather than merging, because
+            // a moved symbol's old path must stop counting against the entry forever.
+            entry.fileHashes = opts.filePaths?.length ? hashFilesForFreshness(opts.filePaths) : undefined;
+            entry.symbolHashes = opts.symbols?.length ? hashSymbolsForFreshness(opts.symbols) : undefined;
+            entry.symbolNormalizedHashes = opts.symbols?.length ? hashSymbolsNormalizedForFreshness(opts.symbols) : undefined;
+            for (const p of opts.filePaths || [])
+                retracked.push(rel(path.resolve(p)));
+            for (const s of opts.symbols || [])
+                retracked.push(`${rel(path.resolve(s.filePath))}::${s.symbolName}`);
+        }
+        else {
+            // Re-anchor in place: same files, same symbols, hashes recomputed against current
+            // content. A tracked path that no longer exists hashes to 'MISSING' exactly as it
+            // does at store time, so the entry stays honestly stale instead of being blessed.
+            if (entry.fileHashes) {
+                const paths = Object.keys(entry.fileHashes);
+                entry.fileHashes = hashFilesForFreshness(paths);
+                for (const p of paths)
+                    retracked.push(rel(p));
+            }
+            if (entry.symbolHashes) {
+                const refs = Object.keys(entry.symbolHashes).map((key) => {
+                    const sep = key.lastIndexOf('::');
+                    return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
+                });
+                entry.symbolHashes = hashSymbolsForFreshness(refs);
+                entry.symbolNormalizedHashes = hashSymbolsNormalizedForFreshness(refs);
+                for (const r of refs)
+                    retracked.push(`${rel(r.filePath)}::${r.symbolName}`);
+            }
+            if (entry.claims) {
+                for (const claim of entry.claims) {
+                    if (claim.fileHashes)
+                        claim.fileHashes = hashFilesForFreshness(Object.keys(claim.fileHashes));
+                    if (claim.symbolHashes) {
+                        const refs = Object.keys(claim.symbolHashes).map((key) => {
+                            const sep = key.lastIndexOf('::');
+                            return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
+                        });
+                        claim.symbolHashes = hashSymbolsForFreshness(refs);
+                        claim.symbolNormalizedHashes = hashSymbolsNormalizedForFreshness(refs);
+                    }
+                }
+            }
+        }
+        // Re-stamp git context and the age clock: this memory was just re-verified here, now.
+        const git = currentGitContext();
+        if (git.commit)
+            entry.gitCommit = git.commit;
+        if (git.branch)
+            entry.gitBranch = git.branch;
+        entry.refreshedAt = new Date().toISOString();
+        const claimIndex = buildClaimIndex(this.entries);
+        const { fresh, staleFiles } = checkEntryFreshness(entry, (depId) => resolveFreshnessNode(this.entries, claimIndex, depId));
+        this.scheduleSave();
+        return {
+            ok: true,
+            fresh,
+            retracked,
+            ...(staleFiles.length > 0 ? { stillStale: staleFiles } : {}),
+            message: fresh
+                ? `Memory "${id}" re-anchored to current code and is fresh again.`
+                : `Memory "${id}" re-anchored, but still stale: ${staleFiles.join(', ')}. ` +
+                    `A "derived:" entry here means an upstream memory it was built on is stale — refresh that one first.`,
+        };
+    }
     // ─── Batch verification ─────────────────────────────────────────────────────
     /**
      * Revalidate ids from a prior search_memory/store_memory result via hash-compare only —
@@ -1033,9 +1643,14 @@ export class TheBrainV2 {
      * either way there is nothing left to vouch for it).
      */
     verifyByIds(ids, options = {}) {
+        // An id handed over by a subagent or another MCP client may belong to an entry this
+        // process has never loaded. Without the sync it would come back 'unknown' — which the
+        // caller is told means "purged", and acting on that would be wrong.
+        this.syncIfChanged();
         const claimIndex = buildClaimIndex(this.entries);
         const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
         const semanticDiff = options.semanticDiff === true;
+        const git = currentGitContext();
         return ids.map((id) => {
             const entry = this.entries.get(id);
             if (entry) {
@@ -1043,6 +1658,7 @@ export class TheBrainV2 {
                 return {
                     id,
                     status: fresh ? 'fresh' : 'stale',
+                    ...(fresh ? {} : attributeStaleness(entry, git)),
                     ...(staleFiles.length > 0 ? { staleFiles } : {}),
                     ...(entry.claims ? { claimBreakdown: entry.claims.map((c) => checkClaimFreshness(c, semanticDiff)) } : {}),
                     ...(cosmeticChanges && cosmeticChanges.length > 0 ? { cosmeticChanges } : {}),
@@ -1068,11 +1684,162 @@ export class TheBrainV2 {
      * next search) — see findBlastRadius for the algorithm and cost.
      */
     findBlastRadius(filePath, symbolName) {
+        this.syncIfChanged();
         return findBlastRadius(this.entries, filePath, symbolName);
     }
-    // ─── Stats ────────────────────────────────────────────────────────────────
-    getStats() {
+    // ─── Portability ────────────────────────────────────────────────────────────
+    /**
+     * Serialize memories to a portable NDJSON bundle: a header line, then one entry per line.
+     *
+     * The Brain is a single file under one user's home directory with no way in or out. That
+     * makes it unshareable with a teammate, unmovable to another machine, and unbackupable
+     * except by copying the directory wholesale (which also copies every other project's
+     * memories). This is the smallest thing that fixes all three.
+     */
+    exportBundle(opts = {}) {
+        this.syncIfChanged();
+        const claimIndex = buildClaimIndex(this.entries);
+        const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
+        let skippedStale = 0;
+        const selected = [];
+        for (const entry of this.entries.values()) {
+            // Same unscoped-entry fallback search() uses: an entry written before scoping existed
+            // belongs to no project in particular, so excluding it from every export would quietly
+            // make it unbackupable.
+            if (opts.projectId && entry.projectId !== undefined && entry.projectId !== opts.projectId)
+                continue;
+            if (opts.domain && entry.domain !== opts.domain)
+                continue;
+            if (!opts.includeStale && !checkEntryFreshness(entry, resolveNode).fresh) {
+                skippedStale++;
+                continue;
+            }
+            selected.push(entry);
+        }
+        const header = JSON.stringify({
+            lemmaBrainExport: 1,
+            exportedAt: new Date().toISOString(),
+            count: selected.length,
+            ...(opts.projectId ? { projectId: opts.projectId } : {}),
+            ...(opts.domain ? { domain: opts.domain } : {}),
+        });
+        const body = selected.map(({ terms: _t, ...persisted }) => JSON.stringify(persisted));
+        return { text: [header, ...body].join('\n'), count: selected.length, skippedStale };
+    }
+    /**
+     * Merge a bundle produced by exportBundle into this Brain.
+     *
+     * Import is additive and never destructive: an id already present keeps whichever version
+     * is newer and the higher of both counters (the same rule cross-process merging uses), and
+     * a tombstoned id stays deleted — importing a bundle must not resurrect something the user
+     * deliberately forgot.
+     *
+     * Imported entries keep their origin machine's absolute paths, so most of them will read
+     * as stale here until refresh() re-anchors them. That is the honest outcome: their
+     * evidence genuinely cannot be verified against this checkout, and reporting them as fresh
+     * would be the one failure mode this whole system is built to prevent.
+     */
+    importBundle(text, opts = {}) {
+        this.syncIfChanged();
+        const lines = text.split('\n').filter(Boolean);
+        if (lines.length === 0)
+            return { ok: false, imported: 0, updated: 0, skipped: 0, message: 'Bundle is empty.' };
+        let start = 0;
+        try {
+            const header = JSON.parse(lines[0]);
+            if (header && header.lemmaBrainExport)
+                start = 1;
+        }
+        catch {
+            // No header — treat the whole file as entries. A raw entries.ndjson copied off another
+            // machine is a perfectly reasonable thing to hand this, and rejecting it would be
+            // pedantry rather than safety.
+        }
+        let imported = 0;
+        let updated = 0;
+        let skipped = 0;
+        const markSource = opts.markSource !== false;
+        for (const line of lines.slice(start)) {
+            let entry;
+            try {
+                entry = JSON.parse(line);
+            }
+            catch {
+                skipped++;
+                continue;
+            }
+            if (!entry || typeof entry.id !== 'string' || typeof entry.query !== 'string') {
+                skipped++;
+                continue;
+            }
+            if (this.tombstones.has(entry.id)) {
+                skipped++;
+                continue;
+            }
+            if (!Array.isArray(entry.terms))
+                entry.terms = Object.keys(entry.termFreq || {});
+            if (entry.terms.length === 0) {
+                // A bundle from a version that persisted neither terms nor termFreq — re-tokenize
+                // rather than admit an entry the inverted index could never retrieve.
+                const terms = tokenize(entry.query + ' ' + (entry.response || ''));
+                entry.terms = [...new Set(terms)];
+                entry.termFreq = termFrequencies(terms);
+            }
+            if (markSource)
+                entry.source = 'import';
+            const existing = this.entries.get(entry.id);
+            if (existing) {
+                if (effectiveTime(entry) <= effectiveTime(existing)) {
+                    // Older or same age: keep ours, but never lose feedback the other side collected.
+                    existing.hits = Math.max(existing.hits || 0, entry.hits || 0);
+                    existing.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
+                    skipped++;
+                    continue;
+                }
+                this.dropEntry(existing);
+                entry.hits = Math.max(existing.hits || 0, entry.hits || 0);
+                entry.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
+                updated++;
+            }
+            else {
+                imported++;
+            }
+            this.entries.set(entry.id, entry);
+            for (const term of entry.terms) {
+                let ids = this.invertedIndex.get(term);
+                if (!ids) {
+                    ids = new Set();
+                    this.invertedIndex.set(term, ids);
+                }
+                ids.add(entry.id);
+            }
+            this.bloom.add(entry.query.trim().toLowerCase().substring(0, 200));
+        }
+        this.recalcAvgDocLength();
+        this.evictIfOverCapacity();
+        this.scheduleSave();
         return {
+            ok: true,
+            imported,
+            updated,
+            skipped,
+            message: `Imported ${imported} new memor${imported === 1 ? 'y' : 'ies'}, updated ${updated}, skipped ${skipped}. ` +
+                `Imported memories track the paths of the machine they came from, so expect them to read as stale here ` +
+                `until refresh_memory re-anchors them.`,
+        };
+    }
+    // ─── Stats ────────────────────────────────────────────────────────────────
+    /**
+     * Corpus counters, plus health signals that answer the question counts alone can't:
+     * is this Brain getting better or is it accumulating dead weight?
+     *
+     * Everything is computed from memory except `staleEntries`, which re-hashes every tracked
+     * file and symbol in the corpus and is therefore behind `deep` — on a full Brain that is
+     * thousands of file reads and has no business running on a routine stats call.
+     */
+    getStats(options = {}) {
+        this.syncIfChanged();
+        const base = {
             totalEntries: this.entries.size,
             totalTerms: this.invertedIndex.size,
             avgDocLength: Math.round(this.avgDocLength),
@@ -1080,6 +1847,52 @@ export class TheBrainV2 {
             cacheHits: this.sessionHits,
             cacheMisses: this.sessionMisses,
         };
+        let neverHit = 0;
+        let downvoted = 0;
+        let unscoped = 0;
+        let oldest = '';
+        const bySource = {};
+        const projects = new Set();
+        for (const entry of this.entries.values()) {
+            if (!entry.hits)
+                neverHit++;
+            if (entry.demerits)
+                downvoted++;
+            if (entry.projectId === undefined)
+                unscoped++;
+            else
+                projects.add(entry.projectId);
+            const src = entrySource(entry);
+            bySource[src] = (bySource[src] || 0) + 1;
+            if (entry.timestamp && (!oldest || entry.timestamp < oldest))
+                oldest = entry.timestamp;
+        }
+        base.neverHit = neverHit;
+        base.downvoted = downvoted;
+        base.bySource = bySource;
+        base.projects = projects.size;
+        base.unscopedEntries = unscoped;
+        base.tombstones = this.tombstones.size;
+        if (oldest)
+            base.oldestEntry = oldest;
+        base.corpusBytes = stampOf(ENTRIES_FILE)?.size ?? 0;
+        if (options.deep) {
+            const claimIndex = buildClaimIndex(this.entries);
+            const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
+            let stale = 0;
+            let tracked = 0;
+            for (const entry of this.entries.values()) {
+                const tracksSomething = !!entry.fileHashes || !!entry.symbolHashes || !!entry.derivedFrom || !!entry.claims;
+                if (!tracksSomething)
+                    continue; // untracked entries are fresh by definition, not evidence
+                tracked++;
+                if (!checkEntryFreshness(entry, resolveNode).fresh)
+                    stale++;
+            }
+            base.staleEntries = stale;
+            base.staleRatio = tracked > 0 ? Number((stale / tracked).toFixed(3)) : 0;
+        }
+        return base;
     }
     clear() {
         this.entries.clear();
@@ -1088,6 +1901,9 @@ export class TheBrainV2 {
         this.avgDocLength = 0;
         this.sessionHits = 0;
         this.sessionMisses = 0;
+        this.tombstones.clear();
+        this.diskStamp = null;
+        this.sidecar = null;
         try {
             if (fs.existsSync(ENTRIES_FILE))
                 fs.unlinkSync(ENTRIES_FILE);
@@ -1095,6 +1911,10 @@ export class TheBrainV2 {
                 fs.unlinkSync(INDEX_FILE);
             if (fs.existsSync(META_FILE))
                 fs.unlinkSync(META_FILE);
+            if (fs.existsSync(LOCK_FILE))
+                fs.unlinkSync(LOCK_FILE);
+            if (fs.existsSync(EMBEDDINGS_FILE))
+                fs.unlinkSync(EMBEDDINGS_FILE);
         }
         catch { /* ignore */ }
     }