brain-memory 0.1.0-beta.1 → 0.1.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/tfidf.js ADDED
@@ -0,0 +1,408 @@
1
+ /**
2
+ * Brain Memory — TF-IDF Search Engine
3
+ *
4
+ * Lightweight, zero-dependency TF-IDF implementation for semantic search
5
+ * across brain memories. Builds a search index at memorize time and
6
+ * computes cosine similarity at recall time.
7
+ *
8
+ * This replaces the "agent eyeballs relevance" approach with deterministic
9
+ * math: tokenize → TF-IDF weights → cosine similarity → numeric score.
10
+ */
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+
15
+ const SEARCH_INDEX_FILE = 'search-index.json';
16
+
17
+ // Common English stopwords — filtered out during tokenization
18
+ const STOPWORDS = new Set([
19
+ 'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
20
+ 'of', 'with', 'by', 'from', 'is', 'was', 'are', 'were', 'be', 'been',
21
+ 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
22
+ 'could', 'should', 'may', 'might', 'shall', 'can', 'need', 'must',
23
+ 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'we',
24
+ 'our', 'you', 'your', 'he', 'him', 'his', 'she', 'her', 'they', 'them',
25
+ 'their', 'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how',
26
+ 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
27
+ 'such', 'no', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
28
+ 'just', 'about', 'above', 'after', 'again', 'also', 'any', 'because',
29
+ 'before', 'between', 'during', 'if', 'into', 'then', 'there', 'up',
30
+ 'out', 'over', 'under', 'here', 'as', 'like', 'use', 'used', 'using',
31
+ ]);
32
+
33
+ /**
34
+ * Simple suffix stemmer — strips common English suffixes.
35
+ * Not as thorough as Porter/Snowball but zero-dependency and good enough
36
+ * for matching "connection" → "connect", "pooling" → "pool", etc.
37
+ *
38
+ * @param {string} word - Lowercase word
39
+ * @returns {string} Stemmed word
40
+ */
41
+ function stem(word) {
42
+ if (word.length <= 3) return word;
43
+
44
+ // Two-pass stemmer: strip compound suffixes first, then simple ones.
45
+ // Pass 1: long compound suffixes
46
+ const longSuffixes = [
47
+ 'izations', 'ational', 'fulness', 'ousness', 'iveness',
48
+ 'ization', 'isation', 'tional', 'ations', 'ments',
49
+ 'ation', 'ness', 'ment', 'ence', 'ance', 'ible', 'able',
50
+ 'ally', 'ious', 'eous', 'ings', 'ions',
51
+ 'ity', 'ful', 'ous', 'ive', 'ize', 'ise', 'ion',
52
+ 'ing', 'ies',
53
+ ];
54
+
55
+ for (const suffix of longSuffixes) {
56
+ if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
57
+ return word.slice(0, -suffix.length);
58
+ }
59
+ }
60
+
61
+ // Pass 2: short suffixes (conservative — skip ambiguous word endings)
62
+ if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2);
63
+ if (word.endsWith('er') && word.length > 4) return word.slice(0, -2);
64
+ if (word.endsWith('ly') && word.length > 4) return word.slice(0, -2);
65
+ if (word.endsWith('al') && word.length > 4) return word.slice(0, -2);
66
+ if (word.endsWith('es') && word.length > 4) return word.slice(0, -2);
67
+
68
+ // Strip trailing 's' only for likely plurals (not ss, us, is endings)
69
+ if (word.endsWith('s') && !word.endsWith('ss') && !word.endsWith('us') &&
70
+ !word.endsWith('is') && word.length > 4) {
71
+ return word.slice(0, -1);
72
+ }
73
+
74
+ return word;
75
+ }
76
+
77
+ /**
78
+ * Tokenize text into stemmed terms, filtering stopwords.
79
+ *
80
+ * @param {string} text - Input text
81
+ * @returns {string[]} Array of stemmed tokens
82
+ */
83
+ function tokenize(text) {
84
+ if (!text) return [];
85
+
86
+ return text
87
+ .toLowerCase()
88
+ .replace(/[^a-z0-9\s_-]/g, ' ') // strip punctuation
89
+ .split(/\s+/) // split on whitespace
90
+ .filter((w) => w.length >= 2 && !STOPWORDS.has(w))
91
+ .map(stem);
92
+ }
93
+
94
+ /**
95
+ * Compute term frequencies for a token list.
96
+ *
97
+ * @param {string[]} tokens - Array of tokens
98
+ * @returns {Object} Map of term → count
99
+ */
100
+ function termFrequencies(tokens) {
101
+ const tf = {};
102
+ for (const token of tokens) {
103
+ tf[token] = (tf[token] || 0) + 1;
104
+ }
105
+ return tf;
106
+ }
107
+
108
+ /**
109
+ * Compute TF-IDF vector for a document given corpus-wide IDF values.
110
+ *
111
+ * TF = (term count in doc) / (total terms in doc)
112
+ * IDF = log(N / df) where df = docs containing the term
113
+ *
114
+ * @param {Object} tf - Term frequency map for this document
115
+ * @param {Object} idf - IDF values (term → idf weight)
116
+ * @param {number} docLength - Total number of tokens in document
117
+ * @returns {Object} Sparse TF-IDF vector (term → weight)
118
+ */
119
+ function computeTfidfVector(tf, idf, docLength) {
120
+ const vector = {};
121
+ if (docLength === 0) return vector;
122
+
123
+ for (const [term, count] of Object.entries(tf)) {
124
+ const tfNorm = count / docLength;
125
+ const idfVal = idf[term] || 0;
126
+ const weight = tfNorm * idfVal;
127
+ if (weight > 0) {
128
+ vector[term] = Math.round(weight * 10000) / 10000;
129
+ }
130
+ }
131
+ return vector;
132
+ }
133
+
134
+ /**
135
+ * Cosine similarity between two sparse vectors.
136
+ *
137
+ * @param {Object} a - Sparse vector (term → weight)
138
+ * @param {Object} b - Sparse vector (term → weight)
139
+ * @returns {number} Similarity (0.0-1.0)
140
+ */
141
+ function cosineSimilarity(a, b) {
142
+ let dot = 0;
143
+ let magA = 0;
144
+ let magB = 0;
145
+
146
+ for (const [term, weight] of Object.entries(a)) {
147
+ magA += weight * weight;
148
+ if (b[term]) {
149
+ dot += weight * b[term];
150
+ }
151
+ }
152
+
153
+ for (const weight of Object.values(b)) {
154
+ magB += weight * weight;
155
+ }
156
+
157
+ const denom = Math.sqrt(magA) * Math.sqrt(magB);
158
+ return denom > 0 ? dot / denom : 0;
159
+ }
160
+
161
+ /**
162
+ * Build text content from a memory for indexing.
163
+ * Concatenates title, body, and tags with appropriate weighting
164
+ * (title terms appear 3x, tags 2x, body 1x).
165
+ *
166
+ * @param {Object} memory - Memory object with title, body, tags
167
+ * @returns {string} Combined text for tokenization
168
+ */
169
+ function buildMemoryText(memory) {
170
+ const parts = [];
171
+
172
+ // Title gets 3x weight (repeat it)
173
+ if (memory.title) {
174
+ parts.push(memory.title, memory.title, memory.title);
175
+ }
176
+
177
+ // Tags get 2x weight
178
+ if (memory.tags && Array.isArray(memory.tags)) {
179
+ const tagText = memory.tags.join(' ');
180
+ parts.push(tagText, tagText);
181
+ }
182
+
183
+ // Body gets 1x weight
184
+ if (memory.body) {
185
+ parts.push(memory.body);
186
+ }
187
+
188
+ // Content field (used in some memories)
189
+ if (memory.content && memory.content !== memory.body) {
190
+ parts.push(memory.content);
191
+ }
192
+
193
+ return parts.join(' ');
194
+ }
195
+
196
+ // ─────────────────────────────────────────────────────────
197
+ // Search Index Management
198
+ // ─────────────────────────────────────────────────────────
199
+
200
+ /**
201
+ * Read the search index from disk.
202
+ *
203
+ * @param {string} brainDir - Absolute path to ~/.brain/
204
+ * @returns {Object|null} Search index or null if not found
205
+ */
206
+ function readSearchIndex(brainDir) {
207
+ const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
208
+ try {
209
+ return JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
210
+ } catch (err) {
211
+ if (err.code === 'ENOENT') return null;
212
+ throw err;
213
+ }
214
+ }
215
+
216
+ /**
217
+ * Write the search index to disk (atomically).
218
+ *
219
+ * @param {string} brainDir - Absolute path to ~/.brain/
220
+ * @param {Object} searchIndex - The search index object
221
+ */
222
+ function writeSearchIndex(brainDir, searchIndex) {
223
+ const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
224
+ const crypto = require('crypto');
225
+ const tmpPath = indexPath + '.tmp.' + crypto.randomBytes(4).toString('hex');
226
+ fs.writeFileSync(tmpPath, JSON.stringify(searchIndex, null, 2) + '\n');
227
+ fs.renameSync(tmpPath, indexPath);
228
+ }
229
+
230
+ /**
231
+ * Create a new empty search index.
232
+ *
233
+ * @returns {Object} Empty search index
234
+ */
235
+ function createSearchIndex() {
236
+ return {
237
+ version: 1,
238
+ doc_count: 0,
239
+ documents: {}, // memoryId → { tf: {term→count}, length: N }
240
+ df: {}, // term → number of documents containing it
241
+ };
242
+ }
243
+
244
+ /**
245
+ * Recompute IDF values from the current document frequencies.
246
+ *
247
+ * @param {Object} searchIndex - The search index
248
+ * @returns {Object} IDF map (term → idf weight)
249
+ */
250
+ function computeIdf(searchIndex) {
251
+ const N = searchIndex.doc_count;
252
+ if (N === 0) return {};
253
+
254
+ const idf = {};
255
+ for (const [term, df] of Object.entries(searchIndex.df)) {
256
+ // Smoothed IDF: ensures terms present in all docs still get a small positive weight
257
+ idf[term] = Math.log(1 + N / df);
258
+ }
259
+ return idf;
260
+ }
261
+
262
+ /**
263
+ * Add a memory document to the search index.
264
+ *
265
+ * @param {Object} searchIndex - The search index (mutated)
266
+ * @param {string} memoryId - Memory ID
267
+ * @param {Object} memory - Memory object (title, body, tags, content)
268
+ * @returns {Object} The mutated search index
269
+ */
270
+ function addDocument(searchIndex, memoryId, memory) {
271
+ // Remove old version if re-indexing
272
+ if (searchIndex.documents[memoryId]) {
273
+ removeDocument(searchIndex, memoryId);
274
+ }
275
+
276
+ const text = buildMemoryText(memory);
277
+ const tokens = tokenize(text);
278
+ const tf = termFrequencies(tokens);
279
+
280
+ searchIndex.documents[memoryId] = {
281
+ tf,
282
+ length: tokens.length,
283
+ };
284
+ searchIndex.doc_count++;
285
+
286
+ // Update document frequencies
287
+ for (const term of Object.keys(tf)) {
288
+ searchIndex.df[term] = (searchIndex.df[term] || 0) + 1;
289
+ }
290
+
291
+ return searchIndex;
292
+ }
293
+
294
+ /**
295
+ * Remove a memory document from the search index.
296
+ *
297
+ * @param {Object} searchIndex - The search index (mutated)
298
+ * @param {string} memoryId - Memory ID
299
+ * @returns {Object} The mutated search index
300
+ */
301
+ function removeDocument(searchIndex, memoryId) {
302
+ const doc = searchIndex.documents[memoryId];
303
+ if (!doc) return searchIndex;
304
+
305
+ // Decrement document frequencies
306
+ for (const term of Object.keys(doc.tf)) {
307
+ if (searchIndex.df[term]) {
308
+ searchIndex.df[term]--;
309
+ if (searchIndex.df[term] <= 0) {
310
+ delete searchIndex.df[term];
311
+ }
312
+ }
313
+ }
314
+
315
+ delete searchIndex.documents[memoryId];
316
+ searchIndex.doc_count--;
317
+
318
+ return searchIndex;
319
+ }
320
+
321
+ /**
322
+ * Compute TF-IDF relevance scores for a query against all indexed memories.
323
+ *
324
+ * @param {Object} searchIndex - The search index
325
+ * @param {string} query - The search query
326
+ * @returns {Object} Map of memoryId → relevance score (0.0-1.0)
327
+ */
328
+ function search(searchIndex, query) {
329
+ if (!searchIndex || searchIndex.doc_count === 0) return {};
330
+
331
+ const queryTokens = tokenize(query);
332
+ if (queryTokens.length === 0) return {};
333
+
334
+ const queryTf = termFrequencies(queryTokens);
335
+ const idf = computeIdf(searchIndex);
336
+ const queryVector = computeTfidfVector(queryTf, idf, queryTokens.length);
337
+
338
+ const scores = {};
339
+
340
+ for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
341
+ const docVector = computeTfidfVector(doc.tf, idf, doc.length);
342
+ const sim = cosineSimilarity(queryVector, docVector);
343
+ if (sim > 0) {
344
+ scores[memoryId] = Math.round(sim * 1000) / 1000;
345
+ }
346
+ }
347
+
348
+ return scores;
349
+ }
350
+
351
+ /**
352
+ * Rebuild the entire search index from memory files on disk.
353
+ *
354
+ * @param {string} brainDir - Absolute path to ~/.brain/
355
+ * @param {Object} index - The brain index (index.json)
356
+ * @returns {Object} Fresh search index
357
+ */
358
+ function rebuildIndex(brainDir, index) {
359
+ const searchIndex = createSearchIndex();
360
+
361
+ if (!index || !index.memories) return searchIndex;
362
+
363
+ for (const [memoryId, entry] of Object.entries(index.memories)) {
364
+ // Read the actual memory file for full content
365
+ const memPath = path.join(brainDir, entry.path);
366
+ let body = '';
367
+ let content = '';
368
+ try {
369
+ const raw = fs.readFileSync(memPath, 'utf-8');
370
+ // Extract body from markdown (after frontmatter)
371
+ const fmEnd = raw.indexOf('---', raw.indexOf('---') + 3);
372
+ if (fmEnd !== -1) {
373
+ body = raw.slice(fmEnd + 3).trim();
374
+ }
375
+ } catch { /* skip unreadable files */ }
376
+
377
+ addDocument(searchIndex, memoryId, {
378
+ title: entry.title || '',
379
+ body,
380
+ tags: entry.tags || [],
381
+ content,
382
+ });
383
+ }
384
+
385
+ return searchIndex;
386
+ }
387
+
388
+ module.exports = {
389
+ // Core primitives
390
+ tokenize,
391
+ stem,
392
+ termFrequencies,
393
+ computeTfidfVector,
394
+ cosineSimilarity,
395
+ buildMemoryText,
396
+ // Index management
397
+ createSearchIndex,
398
+ readSearchIndex,
399
+ writeSearchIndex,
400
+ addDocument,
401
+ removeDocument,
402
+ computeIdf,
403
+ search,
404
+ rebuildIndex,
405
+ // Constants
406
+ SEARCH_INDEX_FILE,
407
+ STOPWORDS,
408
+ };
@@ -1,48 +0,0 @@
1
- # Brain Memory — Session End Hook
2
-
3
- This hook is triggered at the end of a coding session. Its purpose is to prompt the agent to consider storing important memories and to save session context.
4
-
5
- ## Behavior
6
-
7
- ### Memory Suggestion
8
-
9
- When a session ends, evaluate whether the session contained any of the following:
10
-
11
- 1. **Decisions** — Architecture choices, technology selections, trade-off resolutions
12
- 2. **Learnings** — New patterns, debugging insights, API discoveries
13
- 3. **Insights** — Realizations about the codebase, project, or process
14
- 4. **Experiences** — Significant events like incidents, deployments, milestones
15
- 5. **Goals** — New objectives discussed or planned
16
-
17
- If the session contained meaningful content in any of these categories, suggest to the user:
18
-
19
- ```
20
- 💡 This session contained notable <type(s)>. Would you like to store them as brain memories?
21
- Run /brain:memorize to capture them before this context is lost.
22
- ```
23
-
24
- ### Session Context Capture
25
-
26
- If `.brain/contexts.json` exists, append a session summary:
27
-
28
- ```json
29
- {
30
- "session_id": "<timestamp-based-id>",
31
- "started": "<session start ISO timestamp>",
32
- "ended": "<session end ISO timestamp>",
33
- "project": "<current project name>",
34
- "topics": ["<key topics discussed>"],
35
- "task_type": "<primary task type: debugging|implementing|designing|reviewing|discussing|learning>",
36
- "memories_created": ["<IDs of memories stored this session>"],
37
- "memories_recalled": ["<IDs of memories retrieved this session>"]
38
- }
39
- ```
40
-
41
- Keep only the last 20 session entries in `contexts.json` to prevent unbounded growth.
42
-
43
- **Rules:**
44
- - Do NOT auto-memorize without user consent
45
- - Do NOT prompt for trivial sessions (quick fixes, typo corrections, simple questions)
46
- - Only suggest when there is genuinely valuable context worth preserving
47
- - Keep the suggestion brief and non-intrusive
48
- - Always save session context to `contexts.json` regardless of whether the user memorizes anything
@@ -1,48 +0,0 @@
1
- # Brain Memory — Session Start Hook
2
-
3
- This hook is triggered at the start of a coding session. Its purpose is to load relevant context from the brain memory system and capture session context for context-dependent recall.
4
-
5
- ## Behavior
6
-
7
- At session start, if `.brain/index.json` exists:
8
-
9
- 1. **Read the index** to understand the current brain state
10
- 2. **Identify high-value memories** — the top 3-5 memories by effective strength that are relevant to the current project
11
- 3. **Silently load context** — internalize these memories so you can reference them naturally during the session
12
- 4. **Do NOT dump memories** — do not print memory contents at session start unless the user asks
13
-
14
- ### Context Capture
15
-
16
- Capture the current session context for context-dependent recall:
17
- - **Project name**: from the current working directory or project config
18
- - **Topics**: inferred from recent files, open issues, or conversation
19
- - **Task type**: will be determined as the session progresses
20
-
21
- This context is used by `/brain:remember` to boost memories that were encoded in a similar context.
22
-
23
- ### Review Queue Check
24
-
25
- Read `.brain/review-queue.json` if it exists. If there are memories due for review:
26
-
27
- ```
28
- 🧠 Brain active — <N> memories loaded (<M> in current project context)
29
- 📋 <X> memories due for review — run /brain:review
30
- ```
31
-
32
- Otherwise:
33
-
34
- ```
35
- 🧠 Brain active — <N> memories loaded (<M> in current project context)
36
- ```
37
-
38
- ### Low-Confidence Alert
39
-
40
- If any frequently-used memories (access_count >= 3) have confidence < 0.5, briefly note:
41
-
42
- ```
43
- ⚠️ <N> frequently-used memories have low confidence — consider verifying
44
- ```
45
-
46
- **The goal is ambient awareness** — you should know about important past decisions, learnings, and preferences without explicitly reciting them. If a situation arises where a past memory is relevant, naturally reference it.
47
-
48
- Keep the status output to 1-3 lines. The user can run `/brain:status` for details.