brain-memory 0.1.0-beta.3 → 0.1.0-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +319 -0
  2. package/README.md +379 -95
  3. package/SECURITY.md +61 -0
  4. package/assets/brand/README.md +28 -0
  5. package/assets/brand/icon-dark.svg +19 -0
  6. package/assets/brand/icon-light.svg +20 -0
  7. package/assets/brand/linkedin_cover.png +0 -0
  8. package/assets/brand/linkedin_cover_light.png +0 -0
  9. package/assets/brand/mark-dark.svg +19 -0
  10. package/assets/brand/mark-light.svg +20 -0
  11. package/assets/brand/png/icon-dark-1024.png +0 -0
  12. package/assets/brand/png/icon-dark-128.png +0 -0
  13. package/assets/brand/png/icon-dark-256.png +0 -0
  14. package/assets/brand/png/icon-dark-512.png +0 -0
  15. package/assets/brand/png/icon-dark-64.png +0 -0
  16. package/assets/brand/png/icon-light-1024.png +0 -0
  17. package/assets/brand/png/icon-light-128.png +0 -0
  18. package/assets/brand/png/icon-light-256.png +0 -0
  19. package/assets/brand/png/icon-light-512.png +0 -0
  20. package/assets/brand/png/icon-light-64.png +0 -0
  21. package/assets/brand/png/mark-dark-1024.png +0 -0
  22. package/assets/brand/png/mark-light-1024.png +0 -0
  23. package/assets/icon.png +0 -0
  24. package/assets/icon.svg +19 -0
  25. package/bin/brain.js +111 -0
  26. package/bin/cloud-sync.js +150 -0
  27. package/bin/forget.js +122 -0
  28. package/bin/install.js +229 -254
  29. package/bin/memorize.js +458 -0
  30. package/bin/pin.js +44 -0
  31. package/bin/recall.js +251 -0
  32. package/bin/reinforce.js +187 -0
  33. package/bin/session-start.js +298 -0
  34. package/bin/skill.js +67 -0
  35. package/bin/unpin.js +38 -0
  36. package/commands/brain/forget.md +245 -15
  37. package/commands/brain/memorize.md +76 -144
  38. package/commands/brain/pin.md +79 -0
  39. package/commands/brain/remember.md +48 -105
  40. package/commands/brain/skills.md +60 -0
  41. package/commands/brain/sleep.md +69 -28
  42. package/commands/brain/status.md +24 -10
  43. package/commands/brain/sync.md +386 -0
  44. package/package.json +22 -8
  45. package/prompts/antigravity.md +180 -0
  46. package/prompts/claude.md +124 -20
  47. package/prompts/openai.md +124 -20
  48. package/prompts/opencode.md +179 -0
  49. package/src/cloud-sync.js +534 -0
  50. package/src/crypto.js +130 -0
  51. package/src/export-import.js +200 -0
  52. package/src/git-sync.js +444 -0
  53. package/src/index-manager.js +219 -18
  54. package/src/installer.js +444 -0
  55. package/src/pinning.js +124 -0
  56. package/src/scorer.js +22 -9
  57. package/src/skills.js +208 -0
  58. package/src/tfidf.js +501 -0
  59. package/commands/brain/consolidate.md +0 -153
  60. package/commands/brain/explore.md +0 -79
  61. package/commands/brain/init.md +0 -157
  62. package/commands/brain/review.md +0 -130
  63. package/commands/brain/sunshine.md +0 -337
  64. package/hooks/session-end.md +0 -48
  65. package/hooks/session-start.md +0 -48
  66. package/prompts/gemini.md +0 -76
package/src/skills.js ADDED
@@ -0,0 +1,208 @@
1
+ /**
2
+ * Brain Memory — Procedural Skills (CoALA Phase 2 + Tier B §10.3)
3
+ *
4
+ * Stores "how to do things" as `_skills/<name>/SKILL.md` folders and serves them
5
+ * via three-level progressive disclosure:
6
+ * L0 — session start advertises only name + description (skills-index.json)
7
+ * L1 — on a matching task, the agent reads the full SKILL.md
8
+ * L2 — referenced resources/ load only at execution
9
+ *
10
+ * Procedural memory strengthens with successful use and weakens on failure
11
+ * (Tier B §10.3), so a bad crystallized skill demotes itself out of the index.
12
+ */
13
+
14
+ const fs = require('fs');
15
+ const path = require('path');
16
+
17
+ const {
18
+ getBrainDir,
19
+ readSkillsIndex,
20
+ writeSkillsIndex,
21
+ validateBrainPath,
22
+ atomicWriteSync,
23
+ } = require('./index-manager');
24
+ const { setFrontmatterFields } = require('./pinning');
25
+
26
+ const SKILLS_DIR = '_skills';
27
+ const DEFAULT_STRENGTH = 0.6;
28
+ // A skill whose failure rate exceeds this drops below the L0 advertisement cut.
29
+ const DEMOTE_FAIL_RATIO = 0.5;
30
+
31
+ /** Normalize a skill name to a filesystem-safe slug. */
32
+ function slug(name) {
33
+ return String(name || '').toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
34
+ }
35
+
36
+ function skillPath(name) {
37
+ return `${SKILLS_DIR}/${slug(name)}/SKILL.md`;
38
+ }
39
+
40
+ function buildSkillFile(skill) {
41
+ const fm = [
42
+ '---',
43
+ `name: ${slug(skill.name)}`,
44
+ `description: ${JSON.stringify(skill.description || '')}`,
45
+ `triggers: [${(skill.triggers || []).map((t) => JSON.stringify(t)).join(', ')}]`,
46
+ 'cognitive_type: procedural',
47
+ `strength: ${skill.strength ?? DEFAULT_STRENGTH}`,
48
+ `last_used: ${skill.last_used ? JSON.stringify(skill.last_used) : 'null'}`,
49
+ `use_count: ${skill.use_count ?? 0}`,
50
+ `fail_count: ${skill.fail_count ?? 0}`,
51
+ '---',
52
+ '',
53
+ ].join('\n');
54
+ return fm + (skill.body || '') + '\n';
55
+ }
56
+
57
+ /** Is this skill advertised at L0? Demoted skills stay on disk but go quiet. */
58
+ function isAdvertised(s) {
59
+ const uses = s.use_count || 0;
60
+ const fails = s.fail_count || 0;
61
+ return !(uses >= 3 && fails / uses > DEMOTE_FAIL_RATIO);
62
+ }
63
+
64
+ /**
65
+ * Create or replace a skill.
66
+ * @param {string} [projectRoot]
67
+ * @param {Object} skill - { name, description, triggers[], body, strength? }
68
+ */
69
+ function addSkill(projectRoot, skill) {
70
+ const name = slug(skill && skill.name);
71
+ if (!name) return { error: 'Skill name required' };
72
+
73
+ const brainDir = getBrainDir(projectRoot);
74
+ const dir = path.join(brainDir, SKILLS_DIR, name);
75
+ validateBrainPath(dir, brainDir);
76
+ fs.mkdirSync(dir, { recursive: true });
77
+
78
+ const record = {
79
+ name,
80
+ description: skill.description || '',
81
+ triggers: skill.triggers || [],
82
+ strength: skill.strength ?? DEFAULT_STRENGTH,
83
+ use_count: 0,
84
+ fail_count: 0,
85
+ last_used: null,
86
+ };
87
+ atomicWriteSync(path.join(dir, 'SKILL.md'), buildSkillFile({ ...record, body: skill.body }));
88
+
89
+ const idx = readSkillsIndex(projectRoot);
90
+ idx.skills = idx.skills.filter((s) => s.name !== name);
91
+ idx.skills.push({ ...record, path: skillPath(name) });
92
+ writeSkillsIndex(idx, projectRoot);
93
+
94
+ return { name, added: true };
95
+ }
96
+
97
+ /** Full advertised index (every stored skill, advertised or demoted). */
98
+ function listSkills(projectRoot) {
99
+ return readSkillsIndex(projectRoot).skills;
100
+ }
101
+
102
+ /** L0 summaries (name + description) for advertised skills only. */
103
+ function advertisedSummaries(projectRoot) {
104
+ return readSkillsIndex(projectRoot).skills
105
+ .filter(isAdvertised)
106
+ .map((s) => ({ name: s.name, description: s.description, triggers: s.triggers || [] }));
107
+ }
108
+
109
+ /** L1: the full SKILL.md body. */
110
+ function showSkill(projectRoot, name) {
111
+ const full = path.join(getBrainDir(projectRoot), skillPath(name));
112
+ try {
113
+ return { name: slug(name), content: fs.readFileSync(full, 'utf-8') };
114
+ } catch (_) {
115
+ return { error: `Skill not found: ${name}` };
116
+ }
117
+ }
118
+
119
+ /**
120
+ * Record a use. Success strengthens; failure weakens and counts toward demotion.
121
+ * @param {Object} [opts] - { failed: boolean }
122
+ */
123
+ function useSkill(projectRoot, name, opts = {}) {
124
+ const idx = readSkillsIndex(projectRoot);
125
+ const s = idx.skills.find((x) => x.name === slug(name));
126
+ if (!s) return { error: `Skill not found: ${name}` };
127
+
128
+ s.use_count = (s.use_count || 0) + 1;
129
+ if (opts.failed) {
130
+ s.fail_count = (s.fail_count || 0) + 1;
131
+ s.strength = Math.max(0, (s.strength ?? DEFAULT_STRENGTH) - 0.10);
132
+ } else {
133
+ const cur = s.strength ?? DEFAULT_STRENGTH;
134
+ s.strength = Math.min(1.0, cur + 0.05 * (1.0 - cur));
135
+ }
136
+ s.last_used = new Date().toISOString();
137
+ writeSkillsIndex(idx, projectRoot);
138
+
139
+ setFrontmatterFields(getBrainDir(projectRoot), skillPath(name), {
140
+ strength: Math.round(s.strength * 1000) / 1000,
141
+ use_count: s.use_count,
142
+ fail_count: s.fail_count,
143
+ last_used: s.last_used,
144
+ });
145
+
146
+ return {
147
+ name: s.name,
148
+ use_count: s.use_count,
149
+ fail_count: s.fail_count,
150
+ strength: Math.round(s.strength * 1000) / 1000,
151
+ advertised: isAdvertised(s),
152
+ };
153
+ }
154
+
155
+ /**
156
+ * Export a stored skill into the host agent's native skill format so it becomes
157
+ * directly executable (CoALA Phase 4 host bridge). Brain stores/distills; the
158
+ * host executes.
159
+ *
160
+ * @param {string} [projectRoot] - Brain filesystem root
161
+ * @param {string} name - Skill name
162
+ * @param {string} [target='claude'] - 'claude' | 'gemini'
163
+ * @param {string} [destRoot=process.cwd()] - Host project root to write into
164
+ */
165
+ function exportSkill(projectRoot, name, target = 'claude', destRoot = process.cwd()) {
166
+ const src = showSkill(projectRoot, name);
167
+ if (src.error) return src;
168
+
169
+ const dests = {
170
+ claude: path.join(destRoot, '.claude', 'skills', slug(name)),
171
+ gemini: path.join(destRoot, '.gemini', 'skills', slug(name)),
172
+ };
173
+ const dir = dests[target];
174
+ if (!dir) return { error: `Unknown target: ${target} (use claude|gemini)` };
175
+
176
+ fs.mkdirSync(dir, { recursive: true });
177
+ const outPath = path.join(dir, 'SKILL.md');
178
+ fs.writeFileSync(outPath, src.content);
179
+ return { name: slug(name), target, path: outPath };
180
+ }
181
+
182
+ function removeSkill(projectRoot, name) {
183
+ const n = slug(name);
184
+ const dir = path.join(getBrainDir(projectRoot), SKILLS_DIR, n);
185
+ let removedDir = false;
186
+ try { fs.rmSync(dir, { recursive: true, force: true }); removedDir = true; } catch (_) { /* ignore */ }
187
+
188
+ const idx = readSkillsIndex(projectRoot);
189
+ const before = idx.skills.length;
190
+ idx.skills = idx.skills.filter((s) => s.name !== n);
191
+ writeSkillsIndex(idx, projectRoot);
192
+
193
+ return { name: n, removed: removedDir || before !== idx.skills.length };
194
+ }
195
+
196
+ module.exports = {
197
+ slug,
198
+ skillPath,
199
+ isAdvertised,
200
+ addSkill,
201
+ listSkills,
202
+ advertisedSummaries,
203
+ showSkill,
204
+ useSkill,
205
+ removeSkill,
206
+ exportSkill,
207
+ DEMOTE_FAIL_RATIO,
208
+ };
package/src/tfidf.js ADDED
@@ -0,0 +1,501 @@
1
+ /**
2
+ * Brain Memory — TF-IDF Search Engine
3
+ *
4
+ * Lightweight, zero-dependency TF-IDF implementation for semantic search
5
+ * across brain memories. Builds a search index at memorize time and
6
+ * computes cosine similarity at recall time.
7
+ *
8
+ * This replaces the "agent eyeballs relevance" approach with deterministic
9
+ * math: tokenize → TF-IDF weights → cosine similarity → numeric score.
10
+ */
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+
15
+ const SEARCH_INDEX_FILE = 'search-index.json';
16
+
17
+ // Common English stopwords — filtered out during tokenization
18
+ const STOPWORDS = new Set([
19
+ 'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
20
+ 'of', 'with', 'by', 'from', 'is', 'was', 'are', 'were', 'be', 'been',
21
+ 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
22
+ 'could', 'should', 'may', 'might', 'shall', 'can', 'need', 'must',
23
+ 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'we',
24
+ 'our', 'you', 'your', 'he', 'him', 'his', 'she', 'her', 'they', 'them',
25
+ 'their', 'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how',
26
+ 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
27
+ 'such', 'no', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
28
+ 'just', 'about', 'above', 'after', 'again', 'also', 'any', 'because',
29
+ 'before', 'between', 'during', 'if', 'into', 'then', 'there', 'up',
30
+ 'out', 'over', 'under', 'here', 'as', 'like', 'use', 'used', 'using',
31
+ ]);
32
+
33
+ /**
34
+ * Simple suffix stemmer — strips common English suffixes.
35
+ * Not as thorough as Porter/Snowball but zero-dependency and good enough
36
+ * for matching "connection" → "connect", "pooling" → "pool", etc.
37
+ *
38
+ * @param {string} word - Lowercase word
39
+ * @returns {string} Stemmed word
40
+ */
41
+ function stem(word) {
42
+ if (word.length <= 3) return word;
43
+
44
+ // Two-pass stemmer: strip compound suffixes first, then simple ones.
45
+ // Pass 1: long compound suffixes
46
+ const longSuffixes = [
47
+ 'izations', 'ational', 'fulness', 'ousness', 'iveness',
48
+ 'ization', 'isation', 'tional', 'ations', 'ments',
49
+ 'ation', 'ness', 'ment', 'ence', 'ance', 'ible', 'able',
50
+ 'ally', 'ious', 'eous', 'ings', 'ions',
51
+ 'ity', 'ful', 'ous', 'ive', 'ize', 'ise', 'ion',
52
+ 'ing', 'ies',
53
+ ];
54
+
55
+ for (const suffix of longSuffixes) {
56
+ if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
57
+ return word.slice(0, -suffix.length);
58
+ }
59
+ }
60
+
61
+ // Pass 2: short suffixes (conservative — skip ambiguous word endings)
62
+ if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2);
63
+ if (word.endsWith('er') && word.length > 4) return word.slice(0, -2);
64
+ if (word.endsWith('ly') && word.length > 4) return word.slice(0, -2);
65
+ if (word.endsWith('al') && word.length > 4) return word.slice(0, -2);
66
+ if (word.endsWith('es') && word.length > 4) return word.slice(0, -2);
67
+
68
+ // Strip trailing 's' only for likely plurals (not ss, us, is endings)
69
+ if (word.endsWith('s') && !word.endsWith('ss') && !word.endsWith('us') &&
70
+ !word.endsWith('is') && word.length > 4) {
71
+ return word.slice(0, -1);
72
+ }
73
+
74
+ return word;
75
+ }
76
+
77
+ /**
78
+ * Tokenize text into stemmed terms, filtering stopwords.
79
+ *
80
+ * @param {string} text - Input text
81
+ * @returns {string[]} Array of stemmed tokens
82
+ */
83
+ function tokenize(text) {
84
+ if (!text) return [];
85
+
86
+ return text
87
+ .toLowerCase()
88
+ .replace(/[^a-z0-9\s_-]/g, ' ') // strip punctuation
89
+ .split(/\s+/) // split on whitespace
90
+ .filter((w) => w.length >= 2 && !STOPWORDS.has(w))
91
+ .map(stem);
92
+ }
93
+
94
+ /**
95
+ * Compute term frequencies for a token list.
96
+ *
97
+ * @param {string[]} tokens - Array of tokens
98
+ * @returns {Object} Map of term → count
99
+ */
100
+ function termFrequencies(tokens) {
101
+ const tf = {};
102
+ for (const token of tokens) {
103
+ tf[token] = (tf[token] || 0) + 1;
104
+ }
105
+ return tf;
106
+ }
107
+
108
+ /**
109
+ * Compute TF-IDF vector for a document given corpus-wide IDF values.
110
+ *
111
+ * TF = (term count in doc) / (total terms in doc)
112
+ * IDF = log(N / df) where df = docs containing the term
113
+ *
114
+ * @param {Object} tf - Term frequency map for this document
115
+ * @param {Object} idf - IDF values (term → idf weight)
116
+ * @param {number} docLength - Total number of tokens in document
117
+ * @returns {Object} Sparse TF-IDF vector (term → weight)
118
+ */
119
+ function computeTfidfVector(tf, idf, docLength) {
120
+ const vector = {};
121
+ if (docLength === 0) return vector;
122
+
123
+ for (const [term, count] of Object.entries(tf)) {
124
+ const tfNorm = count / docLength;
125
+ const idfVal = idf[term] || 0;
126
+ const weight = tfNorm * idfVal;
127
+ if (weight > 0) {
128
+ vector[term] = Math.round(weight * 10000) / 10000;
129
+ }
130
+ }
131
+ return vector;
132
+ }
133
+
134
+ /**
135
+ * Cosine similarity between two sparse vectors.
136
+ *
137
+ * @param {Object} a - Sparse vector (term → weight)
138
+ * @param {Object} b - Sparse vector (term → weight)
139
+ * @returns {number} Similarity (0.0-1.0)
140
+ */
141
+ function cosineSimilarity(a, b) {
142
+ let dot = 0;
143
+ let magA = 0;
144
+ let magB = 0;
145
+
146
+ for (const [term, weight] of Object.entries(a)) {
147
+ magA += weight * weight;
148
+ if (b[term]) {
149
+ dot += weight * b[term];
150
+ }
151
+ }
152
+
153
+ for (const weight of Object.values(b)) {
154
+ magB += weight * weight;
155
+ }
156
+
157
+ const denom = Math.sqrt(magA) * Math.sqrt(magB);
158
+ return denom > 0 ? dot / denom : 0;
159
+ }
160
+
161
+ /**
162
+ * Build text content from a memory for indexing.
163
+ * Concatenates title, body, and tags with appropriate weighting
164
+ * (title terms appear 3x, tags 2x, body 1x).
165
+ *
166
+ * @param {Object} memory - Memory object with title, body, tags
167
+ * @returns {string} Combined text for tokenization
168
+ */
169
+ function buildMemoryText(memory) {
170
+ const parts = [];
171
+
172
+ // Title gets 3x weight (repeat it)
173
+ if (memory.title) {
174
+ parts.push(memory.title, memory.title, memory.title);
175
+ }
176
+
177
+ // Tags get 2x weight
178
+ if (memory.tags && Array.isArray(memory.tags)) {
179
+ const tagText = memory.tags.join(' ');
180
+ parts.push(tagText, tagText);
181
+ }
182
+
183
+ // Body gets 1x weight
184
+ if (memory.body) {
185
+ parts.push(memory.body);
186
+ }
187
+
188
+ // Content field (used in some memories)
189
+ if (memory.content && memory.content !== memory.body) {
190
+ parts.push(memory.content);
191
+ }
192
+
193
+ return parts.join(' ');
194
+ }
195
+
196
+ // ─────────────────────────────────────────────────────────
197
+ // Search Index Management
198
+ // ─────────────────────────────────────────────────────────
199
+
200
+ /**
201
+ * Read the search index from disk.
202
+ *
203
+ * @param {string} brainDir - Absolute path to ~/.brain/
204
+ * @returns {Object|null} Search index or null if not found
205
+ */
206
+ function readSearchIndex(brainDir) {
207
+ const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
208
+ try {
209
+ return JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
210
+ } catch (err) {
211
+ if (err.code === 'ENOENT') return null;
212
+ throw err;
213
+ }
214
+ }
215
+
216
+ /**
217
+ * Write the search index to disk (atomically).
218
+ *
219
+ * @param {string} brainDir - Absolute path to ~/.brain/
220
+ * @param {Object} searchIndex - The search index object
221
+ */
222
+ function writeSearchIndex(brainDir, searchIndex) {
223
+ const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
224
+ const crypto = require('crypto');
225
+ const tmpPath = indexPath + '.tmp.' + crypto.randomBytes(4).toString('hex');
226
+ fs.writeFileSync(tmpPath, JSON.stringify(searchIndex, null, 2) + '\n');
227
+ fs.renameSync(tmpPath, indexPath);
228
+ }
229
+
230
+ /**
231
+ * Create a new empty search index.
232
+ *
233
+ * @returns {Object} Empty search index
234
+ */
235
+ function createSearchIndex() {
236
+ return {
237
+ version: 1,
238
+ doc_count: 0,
239
+ documents: {}, // memoryId → { tf: {term→count}, length: N }
240
+ df: {}, // term → number of documents containing it
241
+ };
242
+ }
243
+
244
+ /**
245
+ * Recompute IDF values from the current document frequencies.
246
+ *
247
+ * @param {Object} searchIndex - The search index
248
+ * @returns {Object} IDF map (term → idf weight)
249
+ */
250
+ function computeIdf(searchIndex) {
251
+ const N = searchIndex.doc_count;
252
+ if (N === 0) return {};
253
+
254
+ const idf = {};
255
+ for (const [term, df] of Object.entries(searchIndex.df)) {
256
+ // Smoothed IDF: ensures terms present in all docs still get a small positive weight
257
+ idf[term] = Math.log(1 + N / df);
258
+ }
259
+ return idf;
260
+ }
261
+
262
+ /**
263
+ * Add a memory document to the search index.
264
+ *
265
+ * @param {Object} searchIndex - The search index (mutated)
266
+ * @param {string} memoryId - Memory ID
267
+ * @param {Object} memory - Memory object (title, body, tags, content)
268
+ * @returns {Object} The mutated search index
269
+ */
270
+ function addDocument(searchIndex, memoryId, memory) {
271
+ // Remove old version if re-indexing
272
+ if (searchIndex.documents[memoryId]) {
273
+ removeDocument(searchIndex, memoryId);
274
+ }
275
+
276
+ const text = buildMemoryText(memory);
277
+ const tokens = tokenize(text);
278
+ const tf = termFrequencies(tokens);
279
+
280
+ searchIndex.documents[memoryId] = {
281
+ tf,
282
+ length: tokens.length,
283
+ };
284
+ searchIndex.doc_count++;
285
+
286
+ // Update document frequencies
287
+ for (const term of Object.keys(tf)) {
288
+ searchIndex.df[term] = (searchIndex.df[term] || 0) + 1;
289
+ }
290
+
291
+ return searchIndex;
292
+ }
293
+
294
+ /**
295
+ * Remove a memory document from the search index.
296
+ *
297
+ * @param {Object} searchIndex - The search index (mutated)
298
+ * @param {string} memoryId - Memory ID
299
+ * @returns {Object} The mutated search index
300
+ */
301
+ function removeDocument(searchIndex, memoryId) {
302
+ const doc = searchIndex.documents[memoryId];
303
+ if (!doc) return searchIndex;
304
+
305
+ // Decrement document frequencies
306
+ for (const term of Object.keys(doc.tf)) {
307
+ if (searchIndex.df[term]) {
308
+ searchIndex.df[term]--;
309
+ if (searchIndex.df[term] <= 0) {
310
+ delete searchIndex.df[term];
311
+ }
312
+ }
313
+ }
314
+
315
+ delete searchIndex.documents[memoryId];
316
+ searchIndex.doc_count--;
317
+
318
+ return searchIndex;
319
+ }
320
+
321
+ /**
322
+ * Compute TF-IDF relevance scores for a query against all indexed memories.
323
+ *
324
+ * @param {Object} searchIndex - The search index
325
+ * @param {string} query - The search query
326
+ * @returns {Object} Map of memoryId → relevance score (0.0-1.0)
327
+ */
328
+ function search(searchIndex, query) {
329
+ if (!searchIndex || searchIndex.doc_count === 0) return {};
330
+
331
+ const queryTokens = tokenize(query);
332
+ if (queryTokens.length === 0) return {};
333
+
334
+ const queryTf = termFrequencies(queryTokens);
335
+ const idf = computeIdf(searchIndex);
336
+ const queryVector = computeTfidfVector(queryTf, idf, queryTokens.length);
337
+
338
+ const scores = {};
339
+
340
+ for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
341
+ const docVector = computeTfidfVector(doc.tf, idf, doc.length);
342
+ const sim = cosineSimilarity(queryVector, docVector);
343
+ if (sim > 0) {
344
+ scores[memoryId] = Math.round(sim * 1000) / 1000;
345
+ }
346
+ }
347
+
348
+ return scores;
349
+ }
350
+
351
+ /**
352
+ * BM25 relevance scoring for a query against all indexed memories.
353
+ *
354
+ * Fixes two TF-IDF-cosine weaknesses that buried long, detailed memories under
355
+ * short distractors: BM25 SATURATES term frequency (k1) and normalizes document
356
+ * length GENTLY (b), instead of dividing TF by full length and cosine by full
357
+ * magnitude (which penalized the richer oracle memories for being longer). The
358
+ * field weighting baked into the index (title 3x, tags 2x via buildMemoryText)
359
+ * is preserved. Scores are normalized to 0..1 by the top score so they blend
360
+ * cleanly into the scorer's recall formula (which expects relevance in [0,1]).
361
+ *
362
+ * @param {Object} searchIndex - The search index
363
+ * @param {string} query - The search query
364
+ * @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
365
+ * @returns {Object} Map of memoryId → normalized relevance score (0.0-1.0)
366
+ */
367
+ function bm25Search(searchIndex, query, opts = {}) {
368
+ if (!searchIndex || searchIndex.doc_count === 0) return {};
369
+
370
+ const queryTokens = tokenize(query);
371
+ if (queryTokens.length === 0) return {};
372
+
373
+ const k1 = opts.k1 != null ? opts.k1 : 1.5;
374
+ const b = opts.b != null ? opts.b : 0.75;
375
+ const N = searchIndex.doc_count;
376
+
377
+ // Average document length (for length normalization).
378
+ let totalLen = 0;
379
+ for (const d of Object.values(searchIndex.documents)) totalLen += d.length || 0;
380
+ const avgdl = N > 0 ? totalLen / N : 0;
381
+
382
+ // BM25 IDF per (deduped) query term, floored at 0 so a term present in nearly
383
+ // every document can't contribute a negative score.
384
+ const qTerms = [...new Set(queryTokens)];
385
+ const idf = {};
386
+ for (const t of qTerms) {
387
+ const df = searchIndex.df[t] || 0;
388
+ idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
389
+ }
390
+
391
+ const raw = {};
392
+ let maxScore = 0;
393
+ for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
394
+ const dl = doc.length || 0;
395
+ let score = 0;
396
+ for (const t of qTerms) {
397
+ const f = doc.tf[t] || 0;
398
+ if (f === 0) continue;
399
+ const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
400
+ score += idf[t] * (f * (k1 + 1)) / denom;
401
+ }
402
+ if (score > 0) {
403
+ raw[memoryId] = score;
404
+ if (score > maxScore) maxScore = score;
405
+ }
406
+ }
407
+
408
+ // Normalize to 0..1 by the top score — preserves ranking and keeps the [0,1]
409
+ // contract the recall scorer's relevance blend expects.
410
+ const scores = {};
411
+ if (maxScore > 0) {
412
+ for (const [id, s] of Object.entries(raw)) {
413
+ scores[id] = Math.round((s / maxScore) * 1000) / 1000;
414
+ }
415
+ }
416
+ return scores;
417
+ }
418
+
419
+ /**
420
+ * Rebuild the entire search index from memory files on disk.
421
+ *
422
+ * @param {string} brainDir - Absolute path to ~/.brain/
423
+ * @param {Object} index - The brain index (index.json)
424
+ * @returns {Object} Fresh search index
425
+ */
426
+ function rebuildIndex(brainDir, index) {
427
+ const searchIndex = createSearchIndex();
428
+
429
+ if (!index || !index.memories) return searchIndex;
430
+
431
+ for (const [memoryId, entry] of Object.entries(index.memories)) {
432
+ // Read the actual memory file for full content
433
+ const memPath = path.join(brainDir, entry.path);
434
+ let body = '';
435
+ let content = '';
436
+ try {
437
+ const raw = fs.readFileSync(memPath, 'utf-8');
438
+ // Extract body from markdown (after frontmatter)
439
+ const fmEnd = raw.indexOf('---', raw.indexOf('---') + 3);
440
+ if (fmEnd !== -1) {
441
+ body = raw.slice(fmEnd + 3).trim();
442
+ }
443
+ } catch { /* skip unreadable files */ }
444
+
445
+ addDocument(searchIndex, memoryId, {
446
+ title: entry.title || '',
447
+ body,
448
+ tags: entry.tags || [],
449
+ content,
450
+ });
451
+ }
452
+
453
+ return searchIndex;
454
+ }
455
+
456
+ /**
457
+ * Is the persisted search index out of sync with index.json?
458
+ *
459
+ * The search index drifts whenever memories change outside memorize's
460
+ * addDocument path (forget, sleep/consolidate, sync pull, manual edit) — and a
461
+ * present-but-empty index silently makes every relevance score 0. Both `recall`
462
+ * and `session-start` must rebuild when the doc SET (not mere presence) drifts.
463
+ *
464
+ * @param {Object|null} searchIndex - parsed search-index.json (or null if absent)
465
+ * @param {Object} index - parsed index.json
466
+ * @returns {boolean} true when a rebuild is required
467
+ */
468
+ function isSearchIndexStale(searchIndex, index) {
469
+ if (!searchIndex || !searchIndex.documents) return true;
470
+ const memIds = Object.keys((index && index.memories) || {});
471
+ const docCount = searchIndex.doc_count != null
472
+ ? searchIndex.doc_count
473
+ : Object.keys(searchIndex.documents).length;
474
+ if (docCount !== memIds.length) return true;
475
+ for (const id of memIds) if (!(id in searchIndex.documents)) return true;
476
+ return false;
477
+ }
478
+
479
+ module.exports = {
480
+ // Core primitives
481
+ tokenize,
482
+ stem,
483
+ termFrequencies,
484
+ computeTfidfVector,
485
+ cosineSimilarity,
486
+ buildMemoryText,
487
+ // Index management
488
+ createSearchIndex,
489
+ readSearchIndex,
490
+ writeSearchIndex,
491
+ addDocument,
492
+ removeDocument,
493
+ computeIdf,
494
+ search,
495
+ bm25Search,
496
+ rebuildIndex,
497
+ isSearchIndexStale,
498
+ // Constants
499
+ SEARCH_INDEX_FILE,
500
+ STOPWORDS,
501
+ };