@nxuss/lemma 1.16.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/bin/init.js +4 -0
  2. package/dist/cjs/mcp/tools.d.ts.map +1 -1
  3. package/dist/cjs/mcp/tools.js +279 -9
  4. package/dist/cjs/mcp/tools.js.map +1 -1
  5. package/dist/cjs/mcp/utils.d.ts.map +1 -1
  6. package/dist/cjs/mcp/utils.js +21 -2
  7. package/dist/cjs/mcp/utils.js.map +1 -1
  8. package/dist/cjs/pr-review/bridge/BrainBridge.js +1 -1
  9. package/dist/cjs/pr-review/bridge/BrainBridge.js.map +1 -1
  10. package/dist/cjs/subconscious/BrainEmbeddings.d.ts +59 -0
  11. package/dist/cjs/subconscious/BrainEmbeddings.d.ts.map +1 -0
  12. package/dist/cjs/subconscious/BrainEmbeddings.js +222 -0
  13. package/dist/cjs/subconscious/BrainEmbeddings.js.map +1 -0
  14. package/dist/cjs/subconscious/GitIngest.js +2 -2
  15. package/dist/cjs/subconscious/GitIngest.js.map +1 -1
  16. package/dist/cjs/subconscious/TheBrainV2.d.ts +284 -2
  17. package/dist/cjs/subconscious/TheBrainV2.d.ts.map +1 -1
  18. package/dist/cjs/subconscious/TheBrainV2.js +871 -46
  19. package/dist/cjs/subconscious/TheBrainV2.js.map +1 -1
  20. package/dist/cjs/utils/ConversationCheckpoint.js +1 -1
  21. package/dist/cjs/utils/ConversationCheckpoint.js.map +1 -1
  22. package/dist/esm/mcp/tools.d.ts.map +1 -1
  23. package/dist/esm/mcp/tools.js +280 -10
  24. package/dist/esm/mcp/tools.js.map +1 -1
  25. package/dist/esm/mcp/utils.d.ts.map +1 -1
  26. package/dist/esm/mcp/utils.js +21 -2
  27. package/dist/esm/mcp/utils.js.map +1 -1
  28. package/dist/esm/pr-review/bridge/BrainBridge.js +1 -1
  29. package/dist/esm/pr-review/bridge/BrainBridge.js.map +1 -1
  30. package/dist/esm/subconscious/BrainEmbeddings.d.ts +59 -0
  31. package/dist/esm/subconscious/BrainEmbeddings.d.ts.map +1 -0
  32. package/dist/esm/subconscious/BrainEmbeddings.js +211 -0
  33. package/dist/esm/subconscious/BrainEmbeddings.js.map +1 -0
  34. package/dist/esm/subconscious/GitIngest.js +2 -2
  35. package/dist/esm/subconscious/GitIngest.js.map +1 -1
  36. package/dist/esm/subconscious/TheBrainV2.d.ts +284 -2
  37. package/dist/esm/subconscious/TheBrainV2.d.ts.map +1 -1
  38. package/dist/esm/subconscious/TheBrainV2.js +866 -46
  39. package/dist/esm/subconscious/TheBrainV2.js.map +1 -1
  40. package/dist/esm/utils/ConversationCheckpoint.js +1 -1
  41. package/dist/esm/utils/ConversationCheckpoint.js.map +1 -1
  42. package/package.json +1 -1
@@ -15,6 +15,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
17
  exports.TheBrainV2 = exports.MAX_DERIVED_DEPTH = exports.BloomFilter = void 0;
18
+ exports.mergeEntryMaps = mergeEntryMaps;
19
+ exports.currentGitContext = currentGitContext;
20
+ exports.resetGitContextCache = resetGitContextCache;
21
+ exports.attributeStaleness = attributeStaleness;
22
+ exports.entrySource = entrySource;
18
23
  exports.deriveProjectId = deriveProjectId;
19
24
  exports.tokenize = tokenize;
20
25
  exports.termFrequencies = termFrequencies;
@@ -30,10 +35,25 @@ const fs_1 = __importDefault(require("fs"));
30
35
  const path_1 = __importDefault(require("path"));
31
36
  const os_1 = __importDefault(require("os"));
32
37
  const crypto_1 = __importDefault(require("crypto"));
38
+ const child_process_1 = require("child_process");
33
39
  const SymbolSurgicalContext_1 = require("../utils/SymbolSurgicalContext");
40
+ const BrainEmbeddings_1 = require("./BrainEmbeddings");
34
41
  // ─── BM25 Constants ───────────────────────────────────────────────────────────
35
42
  const BM25_K1 = 1.5; // Term saturation (1.2-2.0)
36
43
  const BM25_B = 0.75; // Length normalization (0-1)
44
+ // ─── Recency decay ────────────────────────────────────────────────────────────
45
+ //
46
+ // Deliberately gentle and grace-periodded. Age is a weak proxy for "no longer true" —
47
+ // plenty of architectural decisions stay correct for years — so this is sized to break ties
48
+ // between comparable memories, not to bury an old one that is still the best match. A
49
+ // memory that goes stale is caught by hashing, which is the real signal; this only covers
50
+ // the case hashing can't see, where a memory tracks no files at all.
51
+ /** Days a memory is treated as current with no penalty at all. */
52
+ const RECENCY_GRACE_DAYS = 30;
53
+ /** Days past the grace period at which the penalty reaches its cap. */
54
+ const RECENCY_FULL_DECAY_DAYS = 365;
55
+ /** Cap, in the same 0-1 similarity space as the popularity/outcome/demerit adjustments. */
56
+ const MAX_RECENCY_PENALTY = 0.10;
37
57
  // ─── Storage Paths ────────────────────────────────────────────────────────────
38
58
  /**
39
59
  * LEMMA_BRAIN_DIR redirects storage — required so tests never write into the user's real
@@ -44,6 +64,273 @@ const BRAIN_DIR = process.env.LEMMA_BRAIN_DIR || path_1.default.join(os_1.defaul
44
64
  const ENTRIES_FILE = path_1.default.join(BRAIN_DIR, 'entries.ndjson');
45
65
  const INDEX_FILE = path_1.default.join(BRAIN_DIR, 'index.json');
46
66
  const META_FILE = path_1.default.join(BRAIN_DIR, 'meta.json');
67
+ /** Advisory cross-process write lock. See acquireLock() for why it is advisory, not a guarantee. */
68
+ const LOCK_FILE = path_1.default.join(BRAIN_DIR, '.write.lock');
69
+ /** Sidecar vector cache for the optional semantic re-rank. See BrainEmbeddings.ts. */
70
+ const EMBEDDINGS_FILE = path_1.default.join(BRAIN_DIR, 'embeddings.ndjson');
71
+ // ─── Cross-process durability ─────────────────────────────────────────────────
72
+ //
73
+ // Every MCP client session starts its own server process, and each one holds the whole
74
+ // corpus in memory. Before this section existed, save() blind-rewrote entries.ndjson in
75
+ // full: two concurrent sessions meant whichever flushed last silently erased everything the
76
+ // other had stored since it loaded. That is the exact usage pattern the tool descriptions
77
+ // advertise ("memories from ALL your projects"), so it was not a rare race.
78
+ //
79
+ // Three things fix it, and none of them require a database:
80
+ // 1. save() merges what is on disk into memory before writing, so a full rewrite can only
81
+ // ever be a superset — a writer can add, never subtract (except via tombstones).
82
+ // 2. Writes go through a temp file + rename, so a crash mid-write can't truncate the
83
+ // corpus into something load() would discard as "corrupt, start fresh".
84
+ // 3. An advisory lock file serializes the read-merge-write window between processes.
85
+ // Reads get the same benefit through syncIfChanged(): a long-lived server picks up a
86
+ // sibling session's memories on the next search instead of at the next restart.
87
+ /** A lock older than this is assumed to belong to a process that died holding it. */
88
+ const LOCK_STALE_MS = 10000;
89
+ /** How long a writer waits for another process's lock before proceeding regardless. */
90
+ const LOCK_WAIT_MS = 2000;
91
+ /** Deleted ids stop being suppressed after this long — a tombstone list must not grow forever. */
92
+ const TOMBSTONE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
93
+ /** Blocking sleep. save() is synchronous and its callers must never observe a partial corpus. */
94
+ function sleepSync(ms) {
95
+ try {
96
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
97
+ }
98
+ catch {
99
+ // SharedArrayBuffer unavailable (locked-down runtime) — spin instead of failing.
100
+ const until = Date.now() + ms;
101
+ while (Date.now() < until) { /* busy wait */ }
102
+ }
103
+ }
104
+ /**
105
+ * Take the write lock, or give up after LOCK_WAIT_MS and return null.
106
+ *
107
+ * Deliberately advisory and fail-open: a caller that can't get the lock still writes. The
108
+ * merge in save() is what actually prevents data loss; the lock only narrows the window
109
+ * where two mergers interleave. Blocking a memory write forever on a lock — or throwing —
110
+ * would be a worse outcome than the rare interleave it prevents.
111
+ */
112
+ function acquireLock() {
113
+ const deadline = Date.now() + LOCK_WAIT_MS;
114
+ for (;;) {
115
+ try {
116
+ return fs_1.default.openSync(LOCK_FILE, 'wx');
117
+ }
118
+ catch {
119
+ try {
120
+ if (Date.now() - fs_1.default.statSync(LOCK_FILE).mtimeMs > LOCK_STALE_MS) {
121
+ fs_1.default.unlinkSync(LOCK_FILE);
122
+ continue; // retry immediately against the freed lock
123
+ }
124
+ }
125
+ catch {
126
+ continue; // lock vanished between open and stat — the holder just released it
127
+ }
128
+ if (Date.now() >= deadline)
129
+ return null;
130
+ sleepSync(15);
131
+ }
132
+ }
133
+ }
134
+ function releaseLock(fd) {
135
+ if (fd === null)
136
+ return;
137
+ try {
138
+ fs_1.default.closeSync(fd);
139
+ }
140
+ catch { /* already closed */ }
141
+ try {
142
+ fs_1.default.unlinkSync(LOCK_FILE);
143
+ }
144
+ catch { /* already gone */ }
145
+ }
146
+ /**
147
+ * Write via temp file + rename. rename(2) is atomic within a filesystem, so a reader either
148
+ * sees the whole previous file or the whole new one — never a truncated prefix, which
149
+ * load()'s catch-all would have silently turned into an empty Brain.
150
+ */
151
+ function writeFileAtomic(target, data) {
152
+ const tmp = `${target}.tmp.${process.pid}.${Date.now().toString(36)}`;
153
+ try {
154
+ fs_1.default.writeFileSync(tmp, data, 'utf8');
155
+ fs_1.default.renameSync(tmp, target);
156
+ }
157
+ catch (err) {
158
+ try {
159
+ fs_1.default.unlinkSync(tmp);
160
+ }
161
+ catch { /* nothing to clean up */ }
162
+ throw err;
163
+ }
164
+ }
165
+ function stampOf(file) {
166
+ try {
167
+ const s = fs_1.default.statSync(file);
168
+ return { mtimeMs: s.mtimeMs, size: s.size };
169
+ }
170
+ catch {
171
+ return null;
172
+ }
173
+ }
174
+ function sameStamp(a, b) {
175
+ if (a === null || b === null)
176
+ return a === b;
177
+ return a.mtimeMs === b.mtimeMs && a.size === b.size;
178
+ }
179
+ /** Parses entries.ndjson exactly the way load() does, without touching instance state. */
180
+ function readEntriesFile() {
181
+ const out = new Map();
182
+ let raw;
183
+ try {
184
+ raw = fs_1.default.readFileSync(ENTRIES_FILE, 'utf8');
185
+ }
186
+ catch {
187
+ return out;
188
+ }
189
+ for (const line of raw.split('\n')) {
190
+ if (!line)
191
+ continue;
192
+ try {
193
+ const entry = JSON.parse(line);
194
+ if (!entry || typeof entry.id !== 'string')
195
+ continue;
196
+ // Entries written before the format was compacted still carry `terms`; newer ones
197
+ // don't. Rebuilding from termFreq covers both without a migration step.
198
+ if (!Array.isArray(entry.terms))
199
+ entry.terms = Object.keys(entry.termFreq || {});
200
+ out.set(entry.id, entry);
201
+ }
202
+ catch { /* skip corrupt lines */ }
203
+ }
204
+ return out;
205
+ }
206
+ /** Reads the tombstone map straight off disk, for absorbing a peer process's deletions. */
207
+ function readTombstonesFile() {
208
+ try {
209
+ return readTombstones(JSON.parse(fs_1.default.readFileSync(META_FILE, 'utf8')));
210
+ }
211
+ catch {
212
+ return new Map();
213
+ }
214
+ }
215
+ /** Reads the tombstone map out of a parsed meta.json, tolerating older files without one. */
216
+ function readTombstones(meta) {
217
+ const out = new Map();
218
+ const raw = meta?.tombstones;
219
+ if (!raw || typeof raw !== 'object')
220
+ return out;
221
+ for (const [id, at] of Object.entries(raw)) {
222
+ if (typeof at === 'string')
223
+ out.set(id, at);
224
+ }
225
+ return out;
226
+ }
227
+ /** Effective age anchor: a refreshed memory is as young as its last re-verification. */
228
+ function effectiveTime(entry) {
229
+ const t = Date.parse(entry.refreshedAt || entry.timestamp || '');
230
+ return Number.isFinite(t) ? t : 0;
231
+ }
232
+ /**
233
+ * Union two views of the corpus. Never subtracts: an id present in either side survives
234
+ * unless it is tombstoned, because "absent from my copy" and "deleted" are indistinguishable
235
+ * from inside one process, and guessing wrong loses a user's memory permanently.
236
+ *
237
+ * For an id on both sides, the newer version (by refreshedAt/timestamp) supplies the content
238
+ * and the counters take the max of both — `hits` and `demerits` are monotonic tallies that
239
+ * each process accumulated independently, so max is the only merge that doesn't discard
240
+ * feedback one of them collected.
241
+ */
242
+ function mergeEntryMaps(mine, theirs, tombstones = new Set()) {
243
+ const merged = new Map();
244
+ const put = (entry) => {
245
+ if (tombstones.has(entry.id))
246
+ return;
247
+ const existing = merged.get(entry.id);
248
+ if (!existing) {
249
+ merged.set(entry.id, entry);
250
+ return;
251
+ }
252
+ const winner = effectiveTime(entry) > effectiveTime(existing) ? entry : existing;
253
+ merged.set(entry.id, {
254
+ ...winner,
255
+ hits: Math.max(entry.hits || 0, existing.hits || 0),
256
+ demerits: Math.max(entry.demerits || 0, existing.demerits || 0),
257
+ });
258
+ };
259
+ for (const entry of mine.values())
260
+ put(entry);
261
+ for (const entry of theirs.values())
262
+ put(entry);
263
+ return merged;
264
+ }
265
+ // ─── Git context ──────────────────────────────────────────────────────────────
266
+ /**
267
+ * Current HEAD and branch, cached briefly.
268
+ *
269
+ * Read at store time and again when reporting a stale result, so the Brain can tell
270
+ * "someone edited this function" apart from "you switched branches" — a content hash
271
+ * reports both identically, and only the first is a reason to distrust the memory.
272
+ *
273
+ * Cached because a stale result is attributed inside the search result loop: without it,
274
+ * one search over five stale hits would spawn ten git processes.
275
+ */
276
+ const GIT_CONTEXT_TTL_MS = 5000;
277
+ let _gitContextCache = null;
278
+ function currentGitContext(cwd = process.cwd()) {
279
+ const now = Date.now();
280
+ if (_gitContextCache && _gitContextCache.cwd === cwd && now - _gitContextCache.at < GIT_CONTEXT_TTL_MS) {
281
+ return _gitContextCache.ctx;
282
+ }
283
+ let ctx = {};
284
+ try {
285
+ // Not a git repo, no git binary, detached worktree — all of these must produce {} and
286
+ // never throw. Git context is an explanatory nicety; nothing depends on it existing.
287
+ const run = (args) => (0, child_process_1.execSync)(`git ${args}`, { cwd, encoding: 'utf8', timeout: 1500, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
288
+ const commit = run('rev-parse HEAD');
289
+ const branch = run('rev-parse --abbrev-ref HEAD');
290
+ ctx = {
291
+ ...(commit ? { commit } : {}),
292
+ // "HEAD" is what git reports in a detached checkout — not a branch name, so don't
293
+ // record it as one and later claim the user "switched branches" away from it.
294
+ ...(branch && branch !== 'HEAD' ? { branch } : {}),
295
+ };
296
+ }
297
+ catch {
298
+ ctx = {};
299
+ }
300
+ _gitContextCache = { cwd, at: now, ctx };
301
+ return ctx;
302
+ }
303
+ /** Test seam: forget the cached git context so a test can change branches mid-run. */
304
+ function resetGitContextCache() {
305
+ _gitContextCache = null;
306
+ }
307
+ /**
308
+ * Why a stale entry is stale. Pure attribution over data already gathered — it never
309
+ * changes whether something is stale, only how the staleness is explained.
310
+ */
311
+ function attributeStaleness(entry, git) {
312
+ if (entry.gitBranch && git.branch && entry.gitBranch !== git.branch) {
313
+ return { staleCause: 'branch-changed', storedOnBranch: entry.gitBranch };
314
+ }
315
+ return { staleCause: 'content-changed' };
316
+ }
317
+ /**
318
+ * Normalized provenance for an entry, back-deriving one for entries written before `source`
319
+ * existed. Those overloaded the free-text `provider` field with the same information, so
320
+ * reading it here keeps historical stats meaningful instead of a wall of "unknown".
321
+ */
322
+ function entrySource(entry) {
323
+ if (entry.source)
324
+ return entry.source;
325
+ switch (entry.provider) {
326
+ case 'test_oracle_auto': return 'auto-test';
327
+ case 'checkpoint': return 'auto-checkpoint';
328
+ case 'git-commit':
329
+ case 'changelog': return 'auto-git';
330
+ case 'pr-review': return 'auto-pr-review';
331
+ default: return 'unknown';
332
+ }
333
+ }
47
334
  // ─── Project Scoping ──────────────────────────────────────────────────────────
48
335
  /**
49
336
  * Stable identity for the project a memory belongs to.
@@ -635,6 +922,11 @@ class TheBrainV2 {
635
922
  this.sessionMisses = 0;
636
923
  this.dirty = false;
637
924
  this.flushTimer = null;
925
+ /** State of entries.ndjson as of our last read/write — see syncIfChanged(). */
926
+ this.diskStamp = null;
927
+ /** id -> ISO deletion time, for ids that must not come back through a merge. */
928
+ this.tombstones = new Map();
929
+ this.sidecar = null;
638
930
  this.ensureDir();
639
931
  this.load();
640
932
  }
@@ -645,36 +937,23 @@ class TheBrainV2 {
645
937
  }
646
938
  load() {
647
939
  try {
648
- // Load entries from NDJSON
649
- if (fs_1.default.existsSync(ENTRIES_FILE)) {
650
- const lines = fs_1.default.readFileSync(ENTRIES_FILE, 'utf8').split('\n').filter(Boolean);
651
- for (const line of lines) {
652
- try {
653
- const entry = JSON.parse(line);
654
- // Entries written before the format was compacted still carry `terms`; newer
655
- // ones don't. Rebuilding from termFreq covers both without a migration step.
656
- if (!Array.isArray(entry.terms))
657
- entry.terms = Object.keys(entry.termFreq || {});
658
- this.entries.set(entry.id, entry);
659
- }
660
- catch { /* skip corrupt lines */ }
661
- }
662
- }
663
- // Load inverted index
664
- if (fs_1.default.existsSync(INDEX_FILE)) {
665
- const raw = JSON.parse(fs_1.default.readFileSync(INDEX_FILE, 'utf8'));
666
- for (const [term, ids] of Object.entries(raw)) {
667
- this.invertedIndex.set(term, new Set(ids));
668
- }
669
- }
670
- // Load bloom filter and meta
940
+ // Meta first: the tombstone list decides which entries are allowed back in.
671
941
  if (fs_1.default.existsSync(META_FILE)) {
672
942
  const meta = JSON.parse(fs_1.default.readFileSync(META_FILE, 'utf8'));
673
- if (meta.bloom) {
943
+ if (meta.bloom)
674
944
  this.bloom = new BloomFilter(meta.bloom);
675
- }
676
945
  this.avgDocLength = meta.avgDocLength || 0;
946
+ this.tombstones = readTombstones(meta);
677
947
  }
948
+ this.entries = readEntriesFile();
949
+ for (const id of this.tombstones.keys())
950
+ this.entries.delete(id);
951
+ this.diskStamp = stampOf(ENTRIES_FILE);
952
+ // The inverted index is rebuilt from the entries rather than read back from
953
+ // index.json. It is a pure derivation of the entries, so reconstructing it is the
954
+ // only way it can never disagree with them — and a stale or half-written index.json
955
+ // used to be able to hide entries from search entirely while they sat on disk intact.
956
+ this.rebuildIndex();
678
957
  this.recalcAvgDocLength();
679
958
  }
680
959
  catch (err) {
@@ -682,6 +961,52 @@ class TheBrainV2 {
682
961
  this.entries.clear();
683
962
  this.invertedIndex.clear();
684
963
  this.bloom = new BloomFilter();
964
+ this.tombstones = new Map();
965
+ }
966
+ }
967
+ /** Recomputes the inverted index from scratch over the current entries. */
968
+ rebuildIndex() {
969
+ this.invertedIndex = new Map();
970
+ for (const entry of this.entries.values()) {
971
+ for (const term of entry.terms) {
972
+ let ids = this.invertedIndex.get(term);
973
+ if (!ids) {
974
+ ids = new Set();
975
+ this.invertedIndex.set(term, ids);
976
+ }
977
+ ids.add(entry.id);
978
+ }
979
+ }
980
+ }
981
+ /**
982
+ * Pull in anything another process wrote since we last touched the corpus.
983
+ *
984
+ * Called at the top of every read path. Two statSync calls when nothing changed (the
985
+ * overwhelmingly common case) is far below the cost of the search that follows, and it
986
+ * turns sibling sessions from a data-loss hazard into a live shared corpus: a memory
987
+ * stored in one project's session is searchable from another within one tool call.
988
+ *
989
+ * Merges rather than reloads, so memories stored locally but not yet flushed survive.
990
+ */
991
+ syncIfChanged() {
992
+ const stamp = stampOf(ENTRIES_FILE);
993
+ if (sameStamp(stamp, this.diskStamp))
994
+ return;
995
+ try {
996
+ this.absorbPeerTombstones();
997
+ const onDisk = readEntriesFile();
998
+ const before = this.entries.size;
999
+ this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
1000
+ this.diskStamp = stamp;
1001
+ if (this.entries.size !== before || onDisk.size > 0) {
1002
+ this.rebuildIndex();
1003
+ this.recalcAvgDocLength();
1004
+ }
1005
+ }
1006
+ catch {
1007
+ // A read failure here must never break a search: worst case we keep serving the
1008
+ // in-memory corpus, which is exactly the pre-existing behavior.
1009
+ this.diskStamp = stamp;
685
1010
  }
686
1011
  }
687
1012
  recalcAvgDocLength() {
@@ -703,38 +1028,111 @@ class TheBrainV2 {
703
1028
  }, 500);
704
1029
  }
705
1030
  save() {
1031
+ const fd = acquireLock();
706
1032
  try {
707
1033
  this.ensureDir();
1034
+ // Merge before writing. A full rewrite of what this process happens to hold would
1035
+ // erase every memory a sibling session stored since we loaded — the concurrency bug
1036
+ // this whole section exists to close. After the merge the file we write is a superset
1037
+ // of both views, so a writer can only ever add.
1038
+ this.absorbPeerTombstones();
1039
+ const onDisk = readEntriesFile();
1040
+ if (onDisk.size > 0) {
1041
+ this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
1042
+ this.rebuildIndex();
1043
+ this.recalcAvgDocLength();
1044
+ }
1045
+ this.pruneTombstones();
1046
+ // Post-merge, so the file we write respects the cap even when the merge pulled in
1047
+ // entries a peer had already evicted.
1048
+ this.evictIfOverCapacity();
708
1049
  // Write NDJSON entries. `terms` is dropped: it is exactly Object.keys(termFreq),
709
1050
  // and persisting both made the entry file 38% redundant bytes that every session
710
1051
  // re-read at startup. load() reconstructs it.
711
1052
  const ndjson = Array.from(this.entries.values())
712
1053
  .map(({ terms: _terms, ...persisted }) => JSON.stringify(persisted))
713
1054
  .join('\n');
714
- fs_1.default.writeFileSync(ENTRIES_FILE, ndjson, 'utf8');
715
- // Write inverted index
1055
+ writeFileAtomic(ENTRIES_FILE, ndjson);
1056
+ // Write inverted index. load() rebuilds this from the entries rather than reading it
1057
+ // back, so it is now purely an inspection artifact for the dashboard and for anyone
1058
+ // poking at the brain directory — kept because removing a file other tooling may read
1059
+ // is not worth the handful of bytes it saves.
716
1060
  const indexObj = {};
717
1061
  for (const [term, ids] of this.invertedIndex) {
718
1062
  indexObj[term] = Array.from(ids);
719
1063
  }
720
- fs_1.default.writeFileSync(INDEX_FILE, JSON.stringify(indexObj), 'utf8');
1064
+ writeFileAtomic(INDEX_FILE, JSON.stringify(indexObj));
721
1065
  // Write meta
722
- fs_1.default.writeFileSync(META_FILE, JSON.stringify({
1066
+ writeFileAtomic(META_FILE, JSON.stringify({
723
1067
  bloom: this.bloom.serialize(),
724
1068
  avgDocLength: this.avgDocLength,
725
1069
  totalEntries: this.entries.size,
726
1070
  savedAt: new Date().toISOString(),
727
- }), 'utf8');
1071
+ tombstones: Object.fromEntries(this.tombstones),
1072
+ }));
1073
+ this.diskStamp = stampOf(ENTRIES_FILE);
728
1074
  this.dirty = false;
729
1075
  }
730
1076
  catch { /* fail silently */ }
1077
+ finally {
1078
+ releaseLock(fd);
1079
+ }
1080
+ }
1081
+ /**
1082
+ * Pull in deletions made by other processes.
1083
+ *
1084
+ * Tombstones are shared state, not per-process bookkeeping: a peer that deleted a memory
1085
+ * wrote the tombstone to meta.json, and a process that still holds the entry in memory
1086
+ * would otherwise merge it right back on its next flush — undoing a deliberate deletion
1087
+ * from a session that had nothing to do with it.
1088
+ */
1089
+ absorbPeerTombstones() {
1090
+ for (const [id, at] of readTombstonesFile()) {
1091
+ if (!this.tombstones.has(id))
1092
+ this.tombstones.set(id, at);
1093
+ const entry = this.entries.get(id);
1094
+ if (entry)
1095
+ this.dropEntry(entry);
1096
+ }
1097
+ }
1098
+ /**
1099
+ * Records an id as deliberately deleted, so a merge with a peer that still holds it
1100
+ * doesn't resurrect it. Without this, forget() would be undone the moment any other
1101
+ * session flushed, and eviction would thrash forever between two processes.
1102
+ */
1103
+ tombstone(id) {
1104
+ this.tombstones.set(id, new Date().toISOString());
1105
+ }
1106
+ /** Drops tombstones older than TOMBSTONE_TTL_MS — by then no peer still holds the entry. */
1107
+ pruneTombstones() {
1108
+ const cutoff = Date.now() - TOMBSTONE_TTL_MS;
1109
+ for (const [id, at] of this.tombstones) {
1110
+ const t = Date.parse(at);
1111
+ if (!Number.isFinite(t) || t < cutoff)
1112
+ this.tombstones.delete(id);
1113
+ }
1114
+ }
1115
+ /** Removes an entry from the in-memory corpus and the inverted index. No persistence. */
1116
+ dropEntry(entry) {
1117
+ this.entries.delete(entry.id);
1118
+ for (const term of entry.terms) {
1119
+ const ids = this.invertedIndex.get(term);
1120
+ if (!ids)
1121
+ continue;
1122
+ ids.delete(entry.id);
1123
+ if (ids.size === 0)
1124
+ this.invertedIndex.delete(term);
1125
+ }
731
1126
  }
732
1127
  // ─── Store ────────────────────────────────────────────────────────────────
733
1128
  /**
734
1129
  * Store a query+response pair in the brain.
735
1130
  * Returns false if detected as duplicate (>= dupThreshold similarity).
736
1131
  */
737
- store(query, response, provider = 'generic', dupThreshold = 0.92, filePaths, projectId = deriveProjectId(), outcome, symbolRefs, claimInputs, domain, derivedFrom) {
1132
+ store(query, response, provider = 'generic', dupThreshold = 0.92, filePaths, projectId = deriveProjectId(), outcome, symbolRefs, claimInputs, domain, derivedFrom, options = {}) {
1133
+ // Another session may have stored this exact thing since we loaded; without the sync
1134
+ // the dedup check below would miss it and write a second copy.
1135
+ this.syncIfChanged();
738
1136
  // Quick bloom check
739
1137
  const queryKey = query.trim().toLowerCase().substring(0, 200);
740
1138
  if (this.bloom.has(queryKey)) {
@@ -772,11 +1170,20 @@ class TheBrainV2 {
772
1170
  }
773
1171
  const terms = tokenize(query + ' ' + response);
774
1172
  const termFreq = termFrequencies(terms);
1173
+ // Salted with random bytes, not just the clock: the id used to be sha1(queryKey + now),
1174
+ // so two memories sharing a query prefix and landing in the same millisecond produced
1175
+ // the same id and the second silently overwrote the first — a lost memory that nothing
1176
+ // reported. Batch paths (git ingest, importBundle, auto-capture) hit that window.
775
1177
  const id = crypto_1.default
776
1178
  .createHash('sha1')
777
- .update(queryKey + Date.now())
1179
+ .update(queryKey + Date.now() + crypto_1.default.randomBytes(8).toString('hex'))
778
1180
  .digest('hex')
779
1181
  .substring(0, 12);
1182
+ // Recorded so a later stale result can say whether the code changed or the caller
1183
+ // simply moved branches. Never consulted by freshness itself.
1184
+ const git = options.skipGitContext
1185
+ ? { commit: options.gitCommit, branch: options.gitBranch }
1186
+ : currentGitContext();
780
1187
  const entry = {
781
1188
  id,
782
1189
  query: query.trim(),
@@ -784,7 +1191,7 @@ class TheBrainV2 {
784
1191
  terms: [...new Set(terms)],
785
1192
  termFreq,
786
1193
  provider,
787
- timestamp: new Date().toISOString(),
1194
+ timestamp: options.timestamp || new Date().toISOString(),
788
1195
  hits: 0,
789
1196
  charCount: query.length + response.length,
790
1197
  projectId,
@@ -795,6 +1202,9 @@ class TheBrainV2 {
795
1202
  claims: claimInputs && claimInputs.length > 0 ? buildClaims(claimInputs) : undefined,
796
1203
  domain,
797
1204
  derivedFrom: derivedFrom && derivedFrom.length > 0 ? derivedFrom : undefined,
1205
+ source: options.source || 'manual',
1206
+ ...(git.commit ? { gitCommit: git.commit } : {}),
1207
+ ...(git.branch ? { gitBranch: git.branch } : {}),
798
1208
  };
799
1209
  this.entries.set(id, entry);
800
1210
  // Update inverted index
@@ -809,7 +1219,7 @@ class TheBrainV2 {
809
1219
  this.recalcAvgDocLength();
810
1220
  this.evictIfOverCapacity();
811
1221
  this.scheduleSave();
812
- return { stored: true, reason: 'Stored successfully', conflicts };
1222
+ return { stored: true, reason: 'Stored successfully', id, conflicts };
813
1223
  }
814
1224
  /**
815
1225
  * Evict the lowest-value entries once the Brain is over capacity. Value = hits (proven
@@ -836,15 +1246,11 @@ class TheBrainV2 {
836
1246
  });
837
1247
  const toEvict = ranked.slice(0, TheBrainV2.EVICT_BATCH);
838
1248
  for (const entry of toEvict) {
839
- this.entries.delete(entry.id);
840
- for (const term of entry.terms) {
841
- const ids = this.invertedIndex.get(term);
842
- if (!ids)
843
- continue;
844
- ids.delete(entry.id);
845
- if (ids.size === 0)
846
- this.invertedIndex.delete(term);
847
- }
1249
+ this.dropEntry(entry);
1250
+ // Tombstoned, not just dropped: another session's merge would otherwise hand back
1251
+ // every entry this pass just decided was worthless, and the two processes would
1252
+ // evict-and-resurrect the same 500 entries against each other indefinitely.
1253
+ this.tombstone(entry.id);
848
1254
  }
849
1255
  this.recalcAvgDocLength();
850
1256
  }
@@ -856,6 +1262,8 @@ class TheBrainV2 {
856
1262
  search(query, limit = 5, minSimilarity = 0, options = {}) {
857
1263
  // Internal callers (dedup) must not move the counters the savings ledger reports.
858
1264
  const countStats = options.countStats !== false;
1265
+ // Pick up anything a sibling session stored since the last read. Two statSync calls.
1266
+ this.syncIfChanged();
859
1267
  if (this.entries.size === 0) {
860
1268
  if (countStats)
861
1269
  this.sessionMisses++;
@@ -946,6 +1354,16 @@ class TheBrainV2 {
946
1354
  // popularity prior above rather than just canceling it out.
947
1355
  if (s.entry.demerits)
948
1356
  similarity -= Math.min(s.entry.demerits * 0.06, 0.25);
1357
+ // Age decay. Every other signal here is about the memory's track record; none of them
1358
+ // notice that the codebase it describes has been rewritten twice since. A confirmed
1359
+ // memory from a year ago is not as likely to be current as yesterday's, and until now
1360
+ // they ranked identically. Kept in the same small, additive register as the rest — it
1361
+ // demotes among relevant candidates, it can't bury one. `refreshedAt` resets the clock,
1362
+ // which is the point of refresh(): a re-verified memory really is current again.
1363
+ const ageDays = (Date.now() - effectiveTime(s.entry)) / 86400000;
1364
+ if (Number.isFinite(ageDays) && ageDays > RECENCY_GRACE_DAYS) {
1365
+ similarity -= Math.min((ageDays - RECENCY_GRACE_DAYS) / RECENCY_FULL_DECAY_DAYS, 1) * MAX_RECENCY_PENALTY;
1366
+ }
949
1367
  return { entry: s.entry, similarity: Math.max(0, similarity) };
950
1368
  });
951
1369
  // 6. Sort and filter
@@ -953,6 +1371,9 @@ class TheBrainV2 {
953
1371
  const claimIndex = buildClaimIndex(this.entries);
954
1372
  const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
955
1373
  const semanticDiff = options.semanticDiff === true;
1374
+ // Resolved once per search, not per stale hit — five stale results would otherwise
1375
+ // spawn ten git processes for one query.
1376
+ const git = currentGitContext();
956
1377
  const results = combined
957
1378
  .filter(r => r.similarity >= minSimilarity)
958
1379
  .slice(0, limit)
@@ -967,7 +1388,7 @@ class TheBrainV2 {
967
1388
  provider: r.entry.provider,
968
1389
  timestamp: r.entry.timestamp,
969
1390
  fresh,
970
- ...(fresh ? {} : { staleFiles }),
1391
+ ...(fresh ? {} : { staleFiles, ...attributeStaleness(r.entry, git) }),
971
1392
  ...(r.entry.outcome ? { outcome: r.entry.outcome } : {}),
972
1393
  ...(claims && claims.length > 0 ? { claims } : {}),
973
1394
  ...(r.entry.domain ? { domain: r.entry.domain } : {}),
@@ -995,6 +1416,69 @@ class TheBrainV2 {
995
1416
  * Entries belonging to a project, including the unscoped ones written before entries
996
1417
  * carried a projectId — same fallback rule search() uses, so counts and results agree.
997
1418
  */
1419
+ /**
1420
+ * search(), with an optional semantic re-rank layered on top.
1421
+ *
1422
+ * BM25 still does the retrieving — this only reorders what it already found, and only
1423
+ * when LEMMA_BRAIN_EMBEDDINGS is set and Ollama answers. With the flag off (the default)
1424
+ * this is exactly `search()` plus one boolean check, so callers can use it unconditionally
1425
+ * and no session pays for a feature it hasn't turned on. See BrainEmbeddings.ts.
1426
+ *
1427
+ * The candidate pool is widened before re-ranking: re-ordering the same `limit` results
1428
+ * BM25 already picked can only shuffle them, never surface the memory BM25 ranked 9th
1429
+ * because it happened to use different words — which is the entire point.
1430
+ */
1431
+ async searchHybrid(query, limit = 5, minSimilarity = 0, options = {}) {
1432
+ if (!(0, BrainEmbeddings_1.embeddingsEnabled)())
1433
+ return this.search(query, limit, minSimilarity, options);
1434
+ const poolSize = Math.max(limit * 4, 20);
1435
+ // minSimilarity 0 for the pool: a candidate the re-rank would promote must not be cut
1436
+ // by a lexical floor before the semantic score is ever computed. The floor is applied
1437
+ // again below, against the blended score.
1438
+ const pool = this.search(query, poolSize, 0, { ...options, countStats: false });
1439
+ if (pool.length === 0) {
1440
+ if (options.countStats !== false)
1441
+ this.sessionMisses++;
1442
+ return [];
1443
+ }
1444
+ let ranked = pool;
1445
+ try {
1446
+ const scores = await (0, BrainEmbeddings_1.semanticRerank)(query, pool.map((r) => ({ id: r.id, similarity: r.similarity, text: `${r.query}\n${r.response}` })), this.embeddingSidecar());
1447
+ if (scores) {
1448
+ ranked = pool
1449
+ .map((r) => ({ ...r, similarity: scores.get(r.id) ?? r.similarity }))
1450
+ .sort((a, b) => b.similarity - a.similarity);
1451
+ this.sidecar?.flush(new Set(this.entries.keys()));
1452
+ }
1453
+ }
1454
+ catch {
1455
+ // Re-ranking is an enhancement; a failure in it must never cost the caller the
1456
+ // results BM25 already produced.
1457
+ ranked = pool;
1458
+ }
1459
+ const results = ranked.filter((r) => r.similarity >= minSimilarity).slice(0, limit);
1460
+ // Stats bookkeeping the pool search was told to skip, applied once against the final list
1461
+ // so a hybrid search counts exactly like a lexical one.
1462
+ if (options.countStats !== false) {
1463
+ if (results.length > 0) {
1464
+ this.sessionHits++;
1465
+ const topEntry = this.entries.get(results[0].id);
1466
+ if (topEntry) {
1467
+ topEntry.hits++;
1468
+ this.scheduleSave();
1469
+ }
1470
+ }
1471
+ else {
1472
+ this.sessionMisses++;
1473
+ }
1474
+ }
1475
+ return results;
1476
+ }
1477
+ embeddingSidecar() {
1478
+ if (!this.sidecar)
1479
+ this.sidecar = new BrainEmbeddings_1.EmbeddingSidecar(EMBEDDINGS_FILE);
1480
+ return this.sidecar;
1481
+ }
998
1482
  getEntriesForProject(projectId) {
999
1483
  return Array.from(this.entries.values()).filter((e) => e.projectId === undefined || e.projectId === projectId);
1000
1484
  }
@@ -1035,6 +1519,7 @@ class TheBrainV2 {
1035
1519
  * differently-phrased query tomorrow.
1036
1520
  */
1037
1521
  downvote(id) {
1522
+ this.syncIfChanged();
1038
1523
  const entry = this.entries.get(id);
1039
1524
  if (!entry)
1040
1525
  return { ok: false, message: `No entry with id "${id}" in the Brain.` };
@@ -1042,6 +1527,136 @@ class TheBrainV2 {
1042
1527
  this.scheduleSave();
1043
1528
  return { ok: true, message: `Recorded negative feedback on entry "${id}" (demerits: ${entry.demerits}). It will rank lower and be evicted sooner.` };
1044
1529
  }
1530
+ // ─── Deletion ───────────────────────────────────────────────────────────────
1531
+ /**
1532
+ * Permanently remove one memory.
1533
+ *
1534
+ * The Brain had no way to delete anything: downvote() only demotes, and eviction only
1535
+ * fires at capacity. A memory that is simply wrong, or that captured something that
1536
+ * should never have been stored, had no exit — the only recourse was to downvote it
1537
+ * repeatedly and wait for the corpus to fill up. This is that exit.
1538
+ *
1539
+ * The id is tombstoned as well as dropped, so a concurrent session's merge can't hand it
1540
+ * straight back. The bloom filter cannot un-add a key, so the deleted query may still
1541
+ * register as a possible duplicate later; that costs one extra dedup search and never
1542
+ * produces a wrong answer, which is the right side of that trade for a probabilistic
1543
+ * filter that is rebuilt on the next clear().
1544
+ */
1545
+ forget(id) {
1546
+ this.syncIfChanged();
1547
+ const entry = this.entries.get(id);
1548
+ if (!entry)
1549
+ return { ok: false, message: `No entry with id "${id}" in the Brain.` };
1550
+ this.dropEntry(entry);
1551
+ this.tombstone(id);
1552
+ this.recalcAvgDocLength();
1553
+ this.scheduleSave();
1554
+ return {
1555
+ ok: true,
1556
+ message: `Deleted memory "${id}". It will not come back from another session's copy.`,
1557
+ // Echoed back so a deletion is auditable: the caller (and the user reading the
1558
+ // transcript) can see exactly what was destroyed, since nothing else can recover it.
1559
+ forgotten: { id, query: entry.query, timestamp: entry.timestamp, hits: entry.hits },
1560
+ };
1561
+ }
1562
+ // ─── Re-anchoring ───────────────────────────────────────────────────────────
1563
+ /**
1564
+ * Re-verify a memory against the code as it stands now, keeping its identity.
1565
+ *
1566
+ * Staleness used to be terminal: once tracked evidence changed, an entry was stale
1567
+ * forever, even in the very common case where the insight is still true and the code just
1568
+ * moved. The only workaround was to store a near-duplicate — which store()'s own dedup
1569
+ * guard would often refuse — losing the entry's id, its hit count, its demerits, and every
1570
+ * `derivedFrom` edge pointing at it.
1571
+ *
1572
+ * With no arguments this re-hashes whatever the entry already tracks (and each claim's own
1573
+ * evidence), which is the "yes, I checked, this is still correct" path. Passing filePaths
1574
+ * or symbols instead re-points the entry at new evidence, which is the "the code moved"
1575
+ * path. Either way the caller is asserting the memory is currently true — this tool
1576
+ * records that assertion, it cannot verify it, so it is never called automatically.
1577
+ *
1578
+ * `derivedFrom` dependencies are deliberately not re-anchored: a conclusion inherited from
1579
+ * a memory that is itself stale is exactly what Fase B exists to catch, and silently
1580
+ * clearing that would defeat it. Those come back in `stillStale` instead.
1581
+ */
1582
+ refresh(id, opts = {}) {
1583
+ this.syncIfChanged();
1584
+ const entry = this.entries.get(id);
1585
+ if (!entry)
1586
+ return { ok: false, message: `No entry with id "${id}" in the Brain.` };
1587
+ const retracked = [];
1588
+ const rel = (abs) => {
1589
+ const r = path_1.default.relative(process.cwd(), abs);
1590
+ return r.startsWith('..') ? abs : r;
1591
+ };
1592
+ const hasNewTracking = (opts.filePaths?.length || 0) > 0 || (opts.symbols?.length || 0) > 0;
1593
+ if (hasNewTracking) {
1594
+ // Re-point: the new tracking replaces the old wholesale rather than merging, because
1595
+ // a moved symbol's old path must stop counting against the entry forever.
1596
+ entry.fileHashes = opts.filePaths?.length ? hashFilesForFreshness(opts.filePaths) : undefined;
1597
+ entry.symbolHashes = opts.symbols?.length ? hashSymbolsForFreshness(opts.symbols) : undefined;
1598
+ entry.symbolNormalizedHashes = opts.symbols?.length ? hashSymbolsNormalizedForFreshness(opts.symbols) : undefined;
1599
+ for (const p of opts.filePaths || [])
1600
+ retracked.push(rel(path_1.default.resolve(p)));
1601
+ for (const s of opts.symbols || [])
1602
+ retracked.push(`${rel(path_1.default.resolve(s.filePath))}::${s.symbolName}`);
1603
+ }
1604
+ else {
1605
+ // Re-anchor in place: same files, same symbols, hashes recomputed against current
1606
+ // content. A tracked path that no longer exists hashes to 'MISSING' exactly as it
1607
+ // does at store time, so the entry stays honestly stale instead of being blessed.
1608
+ if (entry.fileHashes) {
1609
+ const paths = Object.keys(entry.fileHashes);
1610
+ entry.fileHashes = hashFilesForFreshness(paths);
1611
+ for (const p of paths)
1612
+ retracked.push(rel(p));
1613
+ }
1614
+ if (entry.symbolHashes) {
1615
+ const refs = Object.keys(entry.symbolHashes).map((key) => {
1616
+ const sep = key.lastIndexOf('::');
1617
+ return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
1618
+ });
1619
+ entry.symbolHashes = hashSymbolsForFreshness(refs);
1620
+ entry.symbolNormalizedHashes = hashSymbolsNormalizedForFreshness(refs);
1621
+ for (const r of refs)
1622
+ retracked.push(`${rel(r.filePath)}::${r.symbolName}`);
1623
+ }
1624
+ if (entry.claims) {
1625
+ for (const claim of entry.claims) {
1626
+ if (claim.fileHashes)
1627
+ claim.fileHashes = hashFilesForFreshness(Object.keys(claim.fileHashes));
1628
+ if (claim.symbolHashes) {
1629
+ const refs = Object.keys(claim.symbolHashes).map((key) => {
1630
+ const sep = key.lastIndexOf('::');
1631
+ return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
1632
+ });
1633
+ claim.symbolHashes = hashSymbolsForFreshness(refs);
1634
+ claim.symbolNormalizedHashes = hashSymbolsNormalizedForFreshness(refs);
1635
+ }
1636
+ }
1637
+ }
1638
+ }
1639
+ // Re-stamp git context and the age clock: this memory was just re-verified here, now.
1640
+ const git = currentGitContext();
1641
+ if (git.commit)
1642
+ entry.gitCommit = git.commit;
1643
+ if (git.branch)
1644
+ entry.gitBranch = git.branch;
1645
+ entry.refreshedAt = new Date().toISOString();
1646
+ const claimIndex = buildClaimIndex(this.entries);
1647
+ const { fresh, staleFiles } = checkEntryFreshness(entry, (depId) => resolveFreshnessNode(this.entries, claimIndex, depId));
1648
+ this.scheduleSave();
1649
+ return {
1650
+ ok: true,
1651
+ fresh,
1652
+ retracked,
1653
+ ...(staleFiles.length > 0 ? { stillStale: staleFiles } : {}),
1654
+ message: fresh
1655
+ ? `Memory "${id}" re-anchored to current code and is fresh again.`
1656
+ : `Memory "${id}" re-anchored, but still stale: ${staleFiles.join(', ')}. ` +
1657
+ `A "derived:" entry here means an upstream memory it was built on is stale — refresh that one first.`,
1658
+ };
1659
+ }
1045
1660
  // ─── Batch verification ─────────────────────────────────────────────────────
1046
1661
  /**
1047
1662
  * Revalidate ids from a prior search_memory/store_memory result via hash-compare only —
@@ -1051,9 +1666,14 @@ class TheBrainV2 {
1051
1666
  * either way there is nothing left to vouch for it).
1052
1667
  */
1053
1668
  verifyByIds(ids, options = {}) {
1669
+ // An id handed over by a subagent or another MCP client may belong to an entry this
1670
+ // process has never loaded. Without the sync it would come back 'unknown' — which the
1671
+ // caller is told means "purged", and acting on that would be wrong.
1672
+ this.syncIfChanged();
1054
1673
  const claimIndex = buildClaimIndex(this.entries);
1055
1674
  const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
1056
1675
  const semanticDiff = options.semanticDiff === true;
1676
+ const git = currentGitContext();
1057
1677
  return ids.map((id) => {
1058
1678
  const entry = this.entries.get(id);
1059
1679
  if (entry) {
@@ -1061,6 +1681,7 @@ class TheBrainV2 {
1061
1681
  return {
1062
1682
  id,
1063
1683
  status: fresh ? 'fresh' : 'stale',
1684
+ ...(fresh ? {} : attributeStaleness(entry, git)),
1064
1685
  ...(staleFiles.length > 0 ? { staleFiles } : {}),
1065
1686
  ...(entry.claims ? { claimBreakdown: entry.claims.map((c) => checkClaimFreshness(c, semanticDiff)) } : {}),
1066
1687
  ...(cosmeticChanges && cosmeticChanges.length > 0 ? { cosmeticChanges } : {}),
@@ -1086,11 +1707,162 @@ class TheBrainV2 {
1086
1707
  * next search) — see findBlastRadius for the algorithm and cost.
1087
1708
  */
1088
1709
  findBlastRadius(filePath, symbolName) {
1710
+ this.syncIfChanged();
1089
1711
  return findBlastRadius(this.entries, filePath, symbolName);
1090
1712
  }
1091
- // ─── Stats ────────────────────────────────────────────────────────────────
1092
- getStats() {
1713
+ // ─── Portability ────────────────────────────────────────────────────────────
1714
+ /**
1715
+ * Serialize memories to a portable NDJSON bundle: a header line, then one entry per line.
1716
+ *
1717
+ * The Brain is a single file under one user's home directory with no way in or out. That
1718
+ * makes it unshareable with a teammate, unmovable to another machine, and unbackupable
1719
+ * except by copying the directory wholesale (which also copies every other project's
1720
+ * memories). This is the smallest thing that fixes all three.
1721
+ */
1722
+ exportBundle(opts = {}) {
1723
+ this.syncIfChanged();
1724
+ const claimIndex = buildClaimIndex(this.entries);
1725
+ const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
1726
+ let skippedStale = 0;
1727
+ const selected = [];
1728
+ for (const entry of this.entries.values()) {
1729
+ // Same unscoped-entry fallback search() uses: an entry written before scoping existed
1730
+ // belongs to no project in particular, so excluding it from every export would quietly
1731
+ // make it unbackupable.
1732
+ if (opts.projectId && entry.projectId !== undefined && entry.projectId !== opts.projectId)
1733
+ continue;
1734
+ if (opts.domain && entry.domain !== opts.domain)
1735
+ continue;
1736
+ if (!opts.includeStale && !checkEntryFreshness(entry, resolveNode).fresh) {
1737
+ skippedStale++;
1738
+ continue;
1739
+ }
1740
+ selected.push(entry);
1741
+ }
1742
+ const header = JSON.stringify({
1743
+ lemmaBrainExport: 1,
1744
+ exportedAt: new Date().toISOString(),
1745
+ count: selected.length,
1746
+ ...(opts.projectId ? { projectId: opts.projectId } : {}),
1747
+ ...(opts.domain ? { domain: opts.domain } : {}),
1748
+ });
1749
+ const body = selected.map(({ terms: _t, ...persisted }) => JSON.stringify(persisted));
1750
+ return { text: [header, ...body].join('\n'), count: selected.length, skippedStale };
1751
+ }
1752
+ /**
1753
+ * Merge a bundle produced by exportBundle into this Brain.
1754
+ *
1755
+ * Import is additive and never destructive: an id already present keeps whichever version
1756
+ * is newer and the higher of both counters (the same rule cross-process merging uses), and
1757
+ * a tombstoned id stays deleted — importing a bundle must not resurrect something the user
1758
+ * deliberately forgot.
1759
+ *
1760
+ * Imported entries keep their origin machine's absolute paths, so most of them will read
1761
+ * as stale here until refresh() re-anchors them. That is the honest outcome: their
1762
+ * evidence genuinely cannot be verified against this checkout, and reporting them as fresh
1763
+ * would be the one failure mode this whole system is built to prevent.
1764
+ */
1765
+ importBundle(text, opts = {}) {
1766
+ this.syncIfChanged();
1767
+ const lines = text.split('\n').filter(Boolean);
1768
+ if (lines.length === 0)
1769
+ return { ok: false, imported: 0, updated: 0, skipped: 0, message: 'Bundle is empty.' };
1770
+ let start = 0;
1771
+ try {
1772
+ const header = JSON.parse(lines[0]);
1773
+ if (header && header.lemmaBrainExport)
1774
+ start = 1;
1775
+ }
1776
+ catch {
1777
+ // No header — treat the whole file as entries. A raw entries.ndjson copied off another
1778
+ // machine is a perfectly reasonable thing to hand this, and rejecting it would be
1779
+ // pedantry rather than safety.
1780
+ }
1781
+ let imported = 0;
1782
+ let updated = 0;
1783
+ let skipped = 0;
1784
+ const markSource = opts.markSource !== false;
1785
+ for (const line of lines.slice(start)) {
1786
+ let entry;
1787
+ try {
1788
+ entry = JSON.parse(line);
1789
+ }
1790
+ catch {
1791
+ skipped++;
1792
+ continue;
1793
+ }
1794
+ if (!entry || typeof entry.id !== 'string' || typeof entry.query !== 'string') {
1795
+ skipped++;
1796
+ continue;
1797
+ }
1798
+ if (this.tombstones.has(entry.id)) {
1799
+ skipped++;
1800
+ continue;
1801
+ }
1802
+ if (!Array.isArray(entry.terms))
1803
+ entry.terms = Object.keys(entry.termFreq || {});
1804
+ if (entry.terms.length === 0) {
1805
+ // A bundle from a version that persisted neither terms nor termFreq — re-tokenize
1806
+ // rather than admit an entry the inverted index could never retrieve.
1807
+ const terms = tokenize(entry.query + ' ' + (entry.response || ''));
1808
+ entry.terms = [...new Set(terms)];
1809
+ entry.termFreq = termFrequencies(terms);
1810
+ }
1811
+ if (markSource)
1812
+ entry.source = 'import';
1813
+ const existing = this.entries.get(entry.id);
1814
+ if (existing) {
1815
+ if (effectiveTime(entry) <= effectiveTime(existing)) {
1816
+ // Older or same age: keep ours, but never lose feedback the other side collected.
1817
+ existing.hits = Math.max(existing.hits || 0, entry.hits || 0);
1818
+ existing.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
1819
+ skipped++;
1820
+ continue;
1821
+ }
1822
+ this.dropEntry(existing);
1823
+ entry.hits = Math.max(existing.hits || 0, entry.hits || 0);
1824
+ entry.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
1825
+ updated++;
1826
+ }
1827
+ else {
1828
+ imported++;
1829
+ }
1830
+ this.entries.set(entry.id, entry);
1831
+ for (const term of entry.terms) {
1832
+ let ids = this.invertedIndex.get(term);
1833
+ if (!ids) {
1834
+ ids = new Set();
1835
+ this.invertedIndex.set(term, ids);
1836
+ }
1837
+ ids.add(entry.id);
1838
+ }
1839
+ this.bloom.add(entry.query.trim().toLowerCase().substring(0, 200));
1840
+ }
1841
+ this.recalcAvgDocLength();
1842
+ this.evictIfOverCapacity();
1843
+ this.scheduleSave();
1093
1844
  return {
1845
+ ok: true,
1846
+ imported,
1847
+ updated,
1848
+ skipped,
1849
+ message: `Imported ${imported} new memor${imported === 1 ? 'y' : 'ies'}, updated ${updated}, skipped ${skipped}. ` +
1850
+ `Imported memories track the paths of the machine they came from, so expect them to read as stale here ` +
1851
+ `until refresh_memory re-anchors them.`,
1852
+ };
1853
+ }
1854
+ // ─── Stats ────────────────────────────────────────────────────────────────
1855
+ /**
1856
+ * Corpus counters, plus health signals that answer the question counts alone can't:
1857
+ * is this Brain getting better or is it accumulating dead weight?
1858
+ *
1859
+ * Everything is computed from memory except `staleEntries`, which re-hashes every tracked
1860
+ * file and symbol in the corpus and is therefore behind `deep` — on a full Brain that is
1861
+ * thousands of file reads and has no business running on a routine stats call.
1862
+ */
1863
+ getStats(options = {}) {
1864
+ this.syncIfChanged();
1865
+ const base = {
1094
1866
  totalEntries: this.entries.size,
1095
1867
  totalTerms: this.invertedIndex.size,
1096
1868
  avgDocLength: Math.round(this.avgDocLength),
@@ -1098,6 +1870,52 @@ class TheBrainV2 {
1098
1870
  cacheHits: this.sessionHits,
1099
1871
  cacheMisses: this.sessionMisses,
1100
1872
  };
1873
+ let neverHit = 0;
1874
+ let downvoted = 0;
1875
+ let unscoped = 0;
1876
+ let oldest = '';
1877
+ const bySource = {};
1878
+ const projects = new Set();
1879
+ for (const entry of this.entries.values()) {
1880
+ if (!entry.hits)
1881
+ neverHit++;
1882
+ if (entry.demerits)
1883
+ downvoted++;
1884
+ if (entry.projectId === undefined)
1885
+ unscoped++;
1886
+ else
1887
+ projects.add(entry.projectId);
1888
+ const src = entrySource(entry);
1889
+ bySource[src] = (bySource[src] || 0) + 1;
1890
+ if (entry.timestamp && (!oldest || entry.timestamp < oldest))
1891
+ oldest = entry.timestamp;
1892
+ }
1893
+ base.neverHit = neverHit;
1894
+ base.downvoted = downvoted;
1895
+ base.bySource = bySource;
1896
+ base.projects = projects.size;
1897
+ base.unscopedEntries = unscoped;
1898
+ base.tombstones = this.tombstones.size;
1899
+ if (oldest)
1900
+ base.oldestEntry = oldest;
1901
+ base.corpusBytes = stampOf(ENTRIES_FILE)?.size ?? 0;
1902
+ if (options.deep) {
1903
+ const claimIndex = buildClaimIndex(this.entries);
1904
+ const resolveNode = (id) => resolveFreshnessNode(this.entries, claimIndex, id);
1905
+ let stale = 0;
1906
+ let tracked = 0;
1907
+ for (const entry of this.entries.values()) {
1908
+ const tracksSomething = !!entry.fileHashes || !!entry.symbolHashes || !!entry.derivedFrom || !!entry.claims;
1909
+ if (!tracksSomething)
1910
+ continue; // untracked entries are fresh by definition, not evidence
1911
+ tracked++;
1912
+ if (!checkEntryFreshness(entry, resolveNode).fresh)
1913
+ stale++;
1914
+ }
1915
+ base.staleEntries = stale;
1916
+ base.staleRatio = tracked > 0 ? Number((stale / tracked).toFixed(3)) : 0;
1917
+ }
1918
+ return base;
1101
1919
  }
1102
1920
  clear() {
1103
1921
  this.entries.clear();
@@ -1106,6 +1924,9 @@ class TheBrainV2 {
1106
1924
  this.avgDocLength = 0;
1107
1925
  this.sessionHits = 0;
1108
1926
  this.sessionMisses = 0;
1927
+ this.tombstones.clear();
1928
+ this.diskStamp = null;
1929
+ this.sidecar = null;
1109
1930
  try {
1110
1931
  if (fs_1.default.existsSync(ENTRIES_FILE))
1111
1932
  fs_1.default.unlinkSync(ENTRIES_FILE);
@@ -1113,6 +1934,10 @@ class TheBrainV2 {
1113
1934
  fs_1.default.unlinkSync(INDEX_FILE);
1114
1935
  if (fs_1.default.existsSync(META_FILE))
1115
1936
  fs_1.default.unlinkSync(META_FILE);
1937
+ if (fs_1.default.existsSync(LOCK_FILE))
1938
+ fs_1.default.unlinkSync(LOCK_FILE);
1939
+ if (fs_1.default.existsSync(EMBEDDINGS_FILE))
1940
+ fs_1.default.unlinkSync(EMBEDDINGS_FILE);
1116
1941
  }
1117
1942
  catch { /* ignore */ }
1118
1943
  }