brain-memory 0.1.0-beta.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +632 -0
- package/README.md +390 -84
- package/SECURITY.md +100 -0
- package/assets/brand/README.md +28 -0
- package/assets/brand/icon-dark.svg +19 -0
- package/assets/brand/icon-light.svg +20 -0
- package/assets/brand/linkedin_cover.png +0 -0
- package/assets/brand/linkedin_cover_light.png +0 -0
- package/assets/brand/mark-dark.svg +19 -0
- package/assets/brand/mark-light.svg +20 -0
- package/assets/brand/png/icon-dark-1024.png +0 -0
- package/assets/brand/png/icon-dark-128.png +0 -0
- package/assets/brand/png/icon-dark-256.png +0 -0
- package/assets/brand/png/icon-dark-512.png +0 -0
- package/assets/brand/png/icon-dark-64.png +0 -0
- package/assets/brand/png/icon-light-1024.png +0 -0
- package/assets/brand/png/icon-light-128.png +0 -0
- package/assets/brand/png/icon-light-256.png +0 -0
- package/assets/brand/png/icon-light-512.png +0 -0
- package/assets/brand/png/icon-light-64.png +0 -0
- package/assets/brand/png/mark-dark-1024.png +0 -0
- package/assets/brand/png/mark-light-1024.png +0 -0
- package/assets/icon.png +0 -0
- package/assets/icon.svg +19 -0
- package/bin/audit.js +93 -0
- package/bin/brain.js +132 -0
- package/bin/cloud-sync.js +150 -0
- package/bin/forget.js +172 -0
- package/bin/import.js +132 -0
- package/bin/install.js +63 -31
- package/bin/memorize.js +670 -0
- package/bin/pin.js +44 -0
- package/bin/recall.js +319 -0
- package/bin/reinforce.js +191 -0
- package/bin/restore.js +122 -0
- package/bin/session-start.js +352 -0
- package/bin/skill.js +67 -0
- package/bin/unpin.js +38 -0
- package/bin/verify.js +186 -0
- package/commands/brain/forget.md +243 -13
- package/commands/brain/import.md +125 -0
- package/commands/brain/memorize.md +108 -130
- package/commands/brain/pin.md +79 -0
- package/commands/brain/remember.md +59 -100
- package/commands/brain/skills.md +60 -0
- package/commands/brain/sleep.md +93 -6
- package/commands/brain/status.md +15 -6
- package/commands/brain/sync.md +181 -17
- package/commands/brain/verify.md +63 -0
- package/integrations/README.md +29 -0
- package/integrations/agentskills/brain-memory/SKILL.md +115 -0
- package/integrations/agentskills/brain-memory/scripts/session-start.mjs +34 -0
- package/package.json +19 -7
- package/prompts/antigravity.md +222 -0
- package/prompts/claude.md +128 -40
- package/prompts/copilot.md +224 -0
- package/prompts/kilo.md +223 -0
- package/prompts/openai.md +133 -45
- package/prompts/opencode.md +221 -0
- package/src/anomaly.js +260 -0
- package/src/audit.js +71 -0
- package/src/cloud-sync.js +672 -0
- package/src/content-lint.js +118 -0
- package/src/export-import.js +45 -7
- package/src/git-sync.js +170 -11
- package/src/harvest.js +562 -0
- package/src/index-manager.js +250 -11
- package/src/installer.js +250 -29
- package/src/pinning.js +146 -0
- package/src/provenance.js +79 -0
- package/src/quarantine.js +111 -0
- package/src/receipt.js +87 -0
- package/src/scorer.js +137 -12
- package/src/skills.js +208 -0
- package/src/temporal.js +228 -0
- package/src/tfidf.js +520 -0
- package/commands/brain/consolidate.md +0 -153
- package/commands/brain/explore.md +0 -79
- package/commands/brain/init.md +0 -149
- package/commands/brain/review.md +0 -130
- package/commands/brain/sunshine.md +0 -337
- package/prompts/gemini.md +0 -134
package/src/tfidf.js
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Brain Memory — TF-IDF Search Engine
|
|
3
|
+
*
|
|
4
|
+
* Lightweight, zero-dependency TF-IDF implementation for semantic search
|
|
5
|
+
* across brain memories. Builds a search index at memorize time and
|
|
6
|
+
* computes cosine similarity at recall time.
|
|
7
|
+
*
|
|
8
|
+
* This replaces the "agent eyeballs relevance" approach with deterministic
|
|
9
|
+
* math: tokenize → TF-IDF weights → cosine similarity → numeric score.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
|
|
15
|
+
const SEARCH_INDEX_FILE = 'search-index.json';
|
|
16
|
+
|
|
17
|
+
// Common English stopwords — filtered out during tokenization
|
|
18
|
+
const STOPWORDS = new Set([
|
|
19
|
+
'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
|
20
|
+
'of', 'with', 'by', 'from', 'is', 'was', 'are', 'were', 'be', 'been',
|
|
21
|
+
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
|
22
|
+
'could', 'should', 'may', 'might', 'shall', 'can', 'need', 'must',
|
|
23
|
+
'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'we',
|
|
24
|
+
'our', 'you', 'your', 'he', 'him', 'his', 'she', 'her', 'they', 'them',
|
|
25
|
+
'their', 'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how',
|
|
26
|
+
'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
|
|
27
|
+
'such', 'no', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
|
|
28
|
+
'just', 'about', 'above', 'after', 'again', 'also', 'any', 'because',
|
|
29
|
+
'before', 'between', 'during', 'if', 'into', 'then', 'there', 'up',
|
|
30
|
+
'out', 'over', 'under', 'here', 'as', 'like', 'use', 'used', 'using',
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Simple suffix stemmer — strips common English suffixes.
|
|
35
|
+
* Not as thorough as Porter/Snowball but zero-dependency and good enough
|
|
36
|
+
* for matching "connection" → "connect", "pooling" → "pool", etc.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} word - Lowercase word
|
|
39
|
+
* @returns {string} Stemmed word
|
|
40
|
+
*/
|
|
41
|
+
function stem(word) {
|
|
42
|
+
if (word.length <= 3) return word;
|
|
43
|
+
|
|
44
|
+
// Two-pass stemmer: strip compound suffixes first, then simple ones.
|
|
45
|
+
// Pass 1: long compound suffixes
|
|
46
|
+
const longSuffixes = [
|
|
47
|
+
'izations', 'ational', 'fulness', 'ousness', 'iveness',
|
|
48
|
+
'ization', 'isation', 'tional', 'ations', 'ments',
|
|
49
|
+
'ation', 'ness', 'ment', 'ence', 'ance', 'ible', 'able',
|
|
50
|
+
'ally', 'ious', 'eous', 'ings', 'ions',
|
|
51
|
+
'ity', 'ful', 'ous', 'ive', 'ize', 'ise', 'ion',
|
|
52
|
+
'ing', 'ies',
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
for (const suffix of longSuffixes) {
|
|
56
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
|
|
57
|
+
return word.slice(0, -suffix.length);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Pass 2: short suffixes (conservative — skip ambiguous word endings)
|
|
62
|
+
if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2);
|
|
63
|
+
if (word.endsWith('er') && word.length > 4) return word.slice(0, -2);
|
|
64
|
+
if (word.endsWith('ly') && word.length > 4) return word.slice(0, -2);
|
|
65
|
+
if (word.endsWith('al') && word.length > 4) return word.slice(0, -2);
|
|
66
|
+
if (word.endsWith('es') && word.length > 4) return word.slice(0, -2);
|
|
67
|
+
|
|
68
|
+
// Strip trailing 's' only for likely plurals (not ss, us, is endings)
|
|
69
|
+
if (word.endsWith('s') && !word.endsWith('ss') && !word.endsWith('us') &&
|
|
70
|
+
!word.endsWith('is') && word.length > 4) {
|
|
71
|
+
return word.slice(0, -1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return word;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Tokenize text into stemmed terms, filtering stopwords.
|
|
79
|
+
*
|
|
80
|
+
* @param {string} text - Input text
|
|
81
|
+
* @returns {string[]} Array of stemmed tokens
|
|
82
|
+
*/
|
|
83
|
+
function tokenize(text) {
|
|
84
|
+
if (!text) return [];
|
|
85
|
+
|
|
86
|
+
return text
|
|
87
|
+
.toLowerCase()
|
|
88
|
+
.replace(/[^a-z0-9\s_-]/g, ' ') // strip punctuation
|
|
89
|
+
.split(/\s+/) // split on whitespace
|
|
90
|
+
.filter((w) => w.length >= 2 && !STOPWORDS.has(w))
|
|
91
|
+
.map(stem);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Compute term frequencies for a token list.
|
|
96
|
+
*
|
|
97
|
+
* @param {string[]} tokens - Array of tokens
|
|
98
|
+
* @returns {Object} Map of term → count
|
|
99
|
+
*/
|
|
100
|
+
function termFrequencies(tokens) {
|
|
101
|
+
const tf = {};
|
|
102
|
+
for (const token of tokens) {
|
|
103
|
+
tf[token] = (tf[token] || 0) + 1;
|
|
104
|
+
}
|
|
105
|
+
return tf;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Compute TF-IDF vector for a document given corpus-wide IDF values.
|
|
110
|
+
*
|
|
111
|
+
* TF = (term count in doc) / (total terms in doc)
|
|
112
|
+
* IDF = log(N / df) where df = docs containing the term
|
|
113
|
+
*
|
|
114
|
+
* @param {Object} tf - Term frequency map for this document
|
|
115
|
+
* @param {Object} idf - IDF values (term → idf weight)
|
|
116
|
+
* @param {number} docLength - Total number of tokens in document
|
|
117
|
+
* @returns {Object} Sparse TF-IDF vector (term → weight)
|
|
118
|
+
*/
|
|
119
|
+
function computeTfidfVector(tf, idf, docLength) {
|
|
120
|
+
const vector = {};
|
|
121
|
+
if (docLength === 0) return vector;
|
|
122
|
+
|
|
123
|
+
for (const [term, count] of Object.entries(tf)) {
|
|
124
|
+
const tfNorm = count / docLength;
|
|
125
|
+
const idfVal = idf[term] || 0;
|
|
126
|
+
const weight = tfNorm * idfVal;
|
|
127
|
+
if (weight > 0) {
|
|
128
|
+
vector[term] = Math.round(weight * 10000) / 10000;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return vector;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Cosine similarity between two sparse vectors.
|
|
136
|
+
*
|
|
137
|
+
* @param {Object} a - Sparse vector (term → weight)
|
|
138
|
+
* @param {Object} b - Sparse vector (term → weight)
|
|
139
|
+
* @returns {number} Similarity (0.0-1.0)
|
|
140
|
+
*/
|
|
141
|
+
function cosineSimilarity(a, b) {
|
|
142
|
+
let dot = 0;
|
|
143
|
+
let magA = 0;
|
|
144
|
+
let magB = 0;
|
|
145
|
+
|
|
146
|
+
for (const [term, weight] of Object.entries(a)) {
|
|
147
|
+
magA += weight * weight;
|
|
148
|
+
if (b[term]) {
|
|
149
|
+
dot += weight * b[term];
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
for (const weight of Object.values(b)) {
|
|
154
|
+
magB += weight * weight;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
158
|
+
return denom > 0 ? dot / denom : 0;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Build text content from a memory for indexing.
|
|
163
|
+
* Concatenates title, body, and tags with appropriate weighting
|
|
164
|
+
* (title terms appear 3x, tags 2x, body 1x).
|
|
165
|
+
*
|
|
166
|
+
* @param {Object} memory - Memory object with title, body, tags
|
|
167
|
+
* @returns {string} Combined text for tokenization
|
|
168
|
+
*/
|
|
169
|
+
function buildMemoryText(memory) {
|
|
170
|
+
const parts = [];
|
|
171
|
+
|
|
172
|
+
// Title gets 3x weight (repeat it)
|
|
173
|
+
if (memory.title) {
|
|
174
|
+
parts.push(memory.title, memory.title, memory.title);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Tags get 2x weight
|
|
178
|
+
if (memory.tags && Array.isArray(memory.tags)) {
|
|
179
|
+
const tagText = memory.tags.join(' ');
|
|
180
|
+
parts.push(tagText, tagText);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Body gets 1x weight
|
|
184
|
+
if (memory.body) {
|
|
185
|
+
parts.push(memory.body);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Content field (used in some memories)
|
|
189
|
+
if (memory.content && memory.content !== memory.body) {
|
|
190
|
+
parts.push(memory.content);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return parts.join(' ');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ─────────────────────────────────────────────────────────
|
|
197
|
+
// Search Index Management
|
|
198
|
+
// ─────────────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Read the search index from disk.
|
|
202
|
+
*
|
|
203
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
204
|
+
* @returns {Object|null} Search index or null if not found
|
|
205
|
+
*/
|
|
206
|
+
function readSearchIndex(brainDir) {
|
|
207
|
+
const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
|
|
208
|
+
try {
|
|
209
|
+
return JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
|
|
210
|
+
} catch (err) {
|
|
211
|
+
if (err.code === 'ENOENT') return null;
|
|
212
|
+
throw err;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Write the search index to disk (atomically).
|
|
218
|
+
*
|
|
219
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
220
|
+
* @param {Object} searchIndex - The search index object
|
|
221
|
+
*/
|
|
222
|
+
function writeSearchIndex(brainDir, searchIndex) {
|
|
223
|
+
const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
|
|
224
|
+
const crypto = require('crypto');
|
|
225
|
+
const tmpPath = indexPath + '.tmp.' + crypto.randomBytes(4).toString('hex');
|
|
226
|
+
fs.writeFileSync(tmpPath, JSON.stringify(searchIndex, null, 2) + '\n');
|
|
227
|
+
fs.renameSync(tmpPath, indexPath);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Create a new empty search index.
|
|
232
|
+
*
|
|
233
|
+
* @returns {Object} Empty search index
|
|
234
|
+
*/
|
|
235
|
+
function createSearchIndex() {
|
|
236
|
+
return {
|
|
237
|
+
version: 1,
|
|
238
|
+
doc_count: 0,
|
|
239
|
+
documents: {}, // memoryId → { tf: {term→count}, length: N }
|
|
240
|
+
df: {}, // term → number of documents containing it
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Recompute IDF values from the current document frequencies.
|
|
246
|
+
*
|
|
247
|
+
* @param {Object} searchIndex - The search index
|
|
248
|
+
* @returns {Object} IDF map (term → idf weight)
|
|
249
|
+
*/
|
|
250
|
+
function computeIdf(searchIndex) {
|
|
251
|
+
const N = searchIndex.doc_count;
|
|
252
|
+
if (N === 0) return {};
|
|
253
|
+
|
|
254
|
+
const idf = {};
|
|
255
|
+
for (const [term, df] of Object.entries(searchIndex.df)) {
|
|
256
|
+
// Smoothed IDF: ensures terms present in all docs still get a small positive weight
|
|
257
|
+
idf[term] = Math.log(1 + N / df);
|
|
258
|
+
}
|
|
259
|
+
return idf;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Add a memory document to the search index.
|
|
264
|
+
*
|
|
265
|
+
* @param {Object} searchIndex - The search index (mutated)
|
|
266
|
+
* @param {string} memoryId - Memory ID
|
|
267
|
+
* @param {Object} memory - Memory object (title, body, tags, content)
|
|
268
|
+
* @returns {Object} The mutated search index
|
|
269
|
+
*/
|
|
270
|
+
function addDocument(searchIndex, memoryId, memory) {
|
|
271
|
+
// Remove old version if re-indexing
|
|
272
|
+
if (searchIndex.documents[memoryId]) {
|
|
273
|
+
removeDocument(searchIndex, memoryId);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const text = buildMemoryText(memory);
|
|
277
|
+
const tokens = tokenize(text);
|
|
278
|
+
const tf = termFrequencies(tokens);
|
|
279
|
+
|
|
280
|
+
searchIndex.documents[memoryId] = {
|
|
281
|
+
tf,
|
|
282
|
+
length: tokens.length,
|
|
283
|
+
};
|
|
284
|
+
searchIndex.doc_count++;
|
|
285
|
+
|
|
286
|
+
// Update document frequencies
|
|
287
|
+
for (const term of Object.keys(tf)) {
|
|
288
|
+
searchIndex.df[term] = (searchIndex.df[term] || 0) + 1;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return searchIndex;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Remove a memory document from the search index.
|
|
296
|
+
*
|
|
297
|
+
* @param {Object} searchIndex - The search index (mutated)
|
|
298
|
+
* @param {string} memoryId - Memory ID
|
|
299
|
+
* @returns {Object} The mutated search index
|
|
300
|
+
*/
|
|
301
|
+
function removeDocument(searchIndex, memoryId) {
|
|
302
|
+
const doc = searchIndex.documents[memoryId];
|
|
303
|
+
if (!doc) return searchIndex;
|
|
304
|
+
|
|
305
|
+
// Decrement document frequencies
|
|
306
|
+
for (const term of Object.keys(doc.tf)) {
|
|
307
|
+
if (searchIndex.df[term]) {
|
|
308
|
+
searchIndex.df[term]--;
|
|
309
|
+
if (searchIndex.df[term] <= 0) {
|
|
310
|
+
delete searchIndex.df[term];
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
delete searchIndex.documents[memoryId];
|
|
316
|
+
searchIndex.doc_count--;
|
|
317
|
+
|
|
318
|
+
return searchIndex;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Compute TF-IDF relevance scores for a query against all indexed memories.
|
|
323
|
+
*
|
|
324
|
+
* @param {Object} searchIndex - The search index
|
|
325
|
+
* @param {string} query - The search query
|
|
326
|
+
* @returns {Object} Map of memoryId → relevance score (0.0-1.0)
|
|
327
|
+
*/
|
|
328
|
+
function search(searchIndex, query) {
|
|
329
|
+
if (!searchIndex || searchIndex.doc_count === 0) return {};
|
|
330
|
+
|
|
331
|
+
const queryTokens = tokenize(query);
|
|
332
|
+
if (queryTokens.length === 0) return {};
|
|
333
|
+
|
|
334
|
+
const queryTf = termFrequencies(queryTokens);
|
|
335
|
+
const idf = computeIdf(searchIndex);
|
|
336
|
+
const queryVector = computeTfidfVector(queryTf, idf, queryTokens.length);
|
|
337
|
+
|
|
338
|
+
const scores = {};
|
|
339
|
+
|
|
340
|
+
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
341
|
+
const docVector = computeTfidfVector(doc.tf, idf, doc.length);
|
|
342
|
+
const sim = cosineSimilarity(queryVector, docVector);
|
|
343
|
+
if (sim > 0) {
|
|
344
|
+
scores[memoryId] = Math.round(sim * 1000) / 1000;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return scores;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* BM25 relevance scoring for a query against all indexed memories.
|
|
353
|
+
*
|
|
354
|
+
* Fixes two TF-IDF-cosine weaknesses that buried long, detailed memories under
|
|
355
|
+
* short distractors: BM25 SATURATES term frequency (k1) and normalizes document
|
|
356
|
+
* length GENTLY (b), instead of dividing TF by full length and cosine by full
|
|
357
|
+
* magnitude (which penalized the richer oracle memories for being longer). The
|
|
358
|
+
* field weighting baked into the index (title 3x, tags 2x via buildMemoryText)
|
|
359
|
+
* is preserved. Scores are normalized to 0..1 by the top score so they blend
|
|
360
|
+
* cleanly into the scorer's recall formula (which expects relevance in [0,1]).
|
|
361
|
+
*
|
|
362
|
+
* Top-score normalization alone is RELATIVE — the best match for a nonsense
|
|
363
|
+
* query would still get relevance 1.0. To keep scores honest in absolute
|
|
364
|
+
* terms, each normalized score is scaled by the doc's IDF-weighted QUERY
|
|
365
|
+
* COVERAGE (share of the query's informative mass the doc actually matched).
|
|
366
|
+
* A doc matching every informative term keeps its score; a doc matching one
|
|
367
|
+
* term of a four-term query is capped near 0.25 no matter how strong that
|
|
368
|
+
* single match is.
|
|
369
|
+
*
|
|
370
|
+
* @param {Object} searchIndex - The search index
|
|
371
|
+
* @param {string} query - The search query
|
|
372
|
+
* @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
|
|
373
|
+
* @returns {Object} Map of memoryId → normalized relevance score (0.0-1.0)
|
|
374
|
+
*/
|
|
375
|
+
function bm25Search(searchIndex, query, opts = {}) {
|
|
376
|
+
if (!searchIndex || searchIndex.doc_count === 0) return {};
|
|
377
|
+
|
|
378
|
+
const queryTokens = tokenize(query);
|
|
379
|
+
if (queryTokens.length === 0) return {};
|
|
380
|
+
|
|
381
|
+
const k1 = opts.k1 != null ? opts.k1 : 1.5;
|
|
382
|
+
const b = opts.b != null ? opts.b : 0.75;
|
|
383
|
+
const N = searchIndex.doc_count;
|
|
384
|
+
|
|
385
|
+
// Average document length (for length normalization).
|
|
386
|
+
let totalLen = 0;
|
|
387
|
+
for (const d of Object.values(searchIndex.documents)) totalLen += d.length || 0;
|
|
388
|
+
const avgdl = N > 0 ? totalLen / N : 0;
|
|
389
|
+
|
|
390
|
+
// BM25 IDF per (deduped) query term, floored at 0 so a term present in nearly
|
|
391
|
+
// every document can't contribute a negative score.
|
|
392
|
+
const qTerms = [...new Set(queryTokens)];
|
|
393
|
+
const idf = {};
|
|
394
|
+
for (const t of qTerms) {
|
|
395
|
+
const df = searchIndex.df[t] || 0;
|
|
396
|
+
idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// Total informative mass of the query. Terms absent from the corpus (df=0)
|
|
400
|
+
// carry high IDF and can never be matched — a query mostly about things the
|
|
401
|
+
// brain doesn't know SHOULD yield low relevance everywhere.
|
|
402
|
+
let idfTotal = 0;
|
|
403
|
+
for (const t of qTerms) idfTotal += idf[t];
|
|
404
|
+
|
|
405
|
+
const raw = {};
|
|
406
|
+
const coverage = {};
|
|
407
|
+
let maxScore = 0;
|
|
408
|
+
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
409
|
+
const dl = doc.length || 0;
|
|
410
|
+
let score = 0;
|
|
411
|
+
let matchedIdf = 0;
|
|
412
|
+
for (const t of qTerms) {
|
|
413
|
+
const f = doc.tf[t] || 0;
|
|
414
|
+
if (f === 0) continue;
|
|
415
|
+
const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
|
|
416
|
+
score += idf[t] * (f * (k1 + 1)) / denom;
|
|
417
|
+
matchedIdf += idf[t];
|
|
418
|
+
}
|
|
419
|
+
if (score > 0) {
|
|
420
|
+
raw[memoryId] = score;
|
|
421
|
+
coverage[memoryId] = idfTotal > 0 ? matchedIdf / idfTotal : 0;
|
|
422
|
+
if (score > maxScore) maxScore = score;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
// Normalize to 0..1 by the top score (preserves ranking, keeps the [0,1]
|
|
427
|
+
// contract the recall scorer's relevance blend expects), then scale by
|
|
428
|
+
// query coverage so the absolute match quality survives normalization.
|
|
429
|
+
const scores = {};
|
|
430
|
+
if (maxScore > 0) {
|
|
431
|
+
for (const [id, s] of Object.entries(raw)) {
|
|
432
|
+
scores[id] = Math.round((s / maxScore) * coverage[id] * 1000) / 1000;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return scores;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Rebuild the entire search index from memory files on disk.
|
|
440
|
+
*
|
|
441
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
442
|
+
* @param {Object} index - The brain index (index.json)
|
|
443
|
+
* @returns {Object} Fresh search index
|
|
444
|
+
*/
|
|
445
|
+
function rebuildIndex(brainDir, index) {
|
|
446
|
+
const searchIndex = createSearchIndex();
|
|
447
|
+
|
|
448
|
+
if (!index || !index.memories) return searchIndex;
|
|
449
|
+
|
|
450
|
+
for (const [memoryId, entry] of Object.entries(index.memories)) {
|
|
451
|
+
// Read the actual memory file for full content
|
|
452
|
+
const memPath = path.join(brainDir, entry.path);
|
|
453
|
+
let body = '';
|
|
454
|
+
let content = '';
|
|
455
|
+
try {
|
|
456
|
+
const raw = fs.readFileSync(memPath, 'utf-8');
|
|
457
|
+
// Extract body from markdown (after frontmatter)
|
|
458
|
+
const fmEnd = raw.indexOf('---', raw.indexOf('---') + 3);
|
|
459
|
+
if (fmEnd !== -1) {
|
|
460
|
+
body = raw.slice(fmEnd + 3).trim();
|
|
461
|
+
}
|
|
462
|
+
} catch { /* skip unreadable files */ }
|
|
463
|
+
|
|
464
|
+
addDocument(searchIndex, memoryId, {
|
|
465
|
+
title: entry.title || '',
|
|
466
|
+
body,
|
|
467
|
+
tags: entry.tags || [],
|
|
468
|
+
content,
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return searchIndex;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Is the persisted search index out of sync with index.json?
|
|
477
|
+
*
|
|
478
|
+
* The search index drifts whenever memories change outside memorize's
|
|
479
|
+
* addDocument path (forget, sleep/consolidate, sync pull, manual edit) — and a
|
|
480
|
+
* present-but-empty index silently makes every relevance score 0. Both `recall`
|
|
481
|
+
* and `session-start` must rebuild when the doc SET (not mere presence) drifts.
|
|
482
|
+
*
|
|
483
|
+
* @param {Object|null} searchIndex - parsed search-index.json (or null if absent)
|
|
484
|
+
* @param {Object} index - parsed index.json
|
|
485
|
+
* @returns {boolean} true when a rebuild is required
|
|
486
|
+
*/
|
|
487
|
+
function isSearchIndexStale(searchIndex, index) {
|
|
488
|
+
if (!searchIndex || !searchIndex.documents) return true;
|
|
489
|
+
const memIds = Object.keys((index && index.memories) || {});
|
|
490
|
+
const docCount = searchIndex.doc_count != null
|
|
491
|
+
? searchIndex.doc_count
|
|
492
|
+
: Object.keys(searchIndex.documents).length;
|
|
493
|
+
if (docCount !== memIds.length) return true;
|
|
494
|
+
for (const id of memIds) if (!(id in searchIndex.documents)) return true;
|
|
495
|
+
return false;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
module.exports = {
|
|
499
|
+
// Core primitives
|
|
500
|
+
tokenize,
|
|
501
|
+
stem,
|
|
502
|
+
termFrequencies,
|
|
503
|
+
computeTfidfVector,
|
|
504
|
+
cosineSimilarity,
|
|
505
|
+
buildMemoryText,
|
|
506
|
+
// Index management
|
|
507
|
+
createSearchIndex,
|
|
508
|
+
readSearchIndex,
|
|
509
|
+
writeSearchIndex,
|
|
510
|
+
addDocument,
|
|
511
|
+
removeDocument,
|
|
512
|
+
computeIdf,
|
|
513
|
+
search,
|
|
514
|
+
bm25Search,
|
|
515
|
+
rebuildIndex,
|
|
516
|
+
isSearchIndexStale,
|
|
517
|
+
// Constants
|
|
518
|
+
SEARCH_INDEX_FILE,
|
|
519
|
+
STOPWORDS,
|
|
520
|
+
};
|
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
# /brain:consolidate — Merge Related Memories
|
|
2
|
-
|
|
3
|
-
You are consolidating memories in the Brain Memory system. Consolidation combines multiple related or weakening memories into a single stronger, more coherent memory — mimicking how human brains merge similar experiences into generalized knowledge.
|
|
4
|
-
|
|
5
|
-
**Target scope:** $ARGUMENTS
|
|
6
|
-
|
|
7
|
-
## Steps
|
|
8
|
-
|
|
9
|
-
### 1. Determine Scope
|
|
10
|
-
|
|
11
|
-
- If $ARGUMENTS specifies a category path, consolidate within that subtree
|
|
12
|
-
- If $ARGUMENTS is empty, scan the entire `~/.brain/` for consolidation candidates
|
|
13
|
-
- If $ARGUMENTS is "auto", perform automatic consolidation based on thresholds
|
|
14
|
-
|
|
15
|
-
### 2. Find Consolidation Candidates
|
|
16
|
-
|
|
17
|
-
Read `index.json` and identify candidate groups using these criteria:
|
|
18
|
-
|
|
19
|
-
**Automatic candidates (any of):**
|
|
20
|
-
- Memories with effective (decayed) strength below `consolidation_threshold` (default: 0.3)
|
|
21
|
-
- Clusters of 3+ memories in the same directory with overlapping tags
|
|
22
|
-
- Memories with the same type in the same category that are older than 30 days
|
|
23
|
-
|
|
24
|
-
**Compute decayed strength for each:**
|
|
25
|
-
```
|
|
26
|
-
days_elapsed = (now - last_accessed) / (1000 * 60 * 60 * 24)
|
|
27
|
-
decayed_strength = strength * (decay_rate ^ days_elapsed)
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
### 3. Group Related Memories
|
|
31
|
-
|
|
32
|
-
Group candidates by:
|
|
33
|
-
1. **Path proximity** — memories in the same or nearby directories
|
|
34
|
-
2. **Tag overlap** — memories sharing 2+ tags
|
|
35
|
-
3. **Type similarity** — same memory type
|
|
36
|
-
4. **Temporal proximity** — created within similar time periods
|
|
37
|
-
|
|
38
|
-
Present the proposed groups to the user:
|
|
39
|
-
|
|
40
|
-
```
|
|
41
|
-
## Consolidation Candidates
|
|
42
|
-
|
|
43
|
-
### Group 1: Kubernetes Deployments (professional/companies/acme/projects/)
|
|
44
|
-
- deployment-v1.md (strength: 0.22, type: experience)
|
|
45
|
-
- deployment-v2.md (strength: 0.18, type: experience)
|
|
46
|
-
- k8s-config-issue.md (strength: 0.15, type: learning)
|
|
47
|
-
→ Proposed: Merge into "kubernetes-deployment-lessons.md" (strength: 0.75)
|
|
48
|
-
|
|
49
|
-
### Group 2: TypeScript Learnings (personal/education/)
|
|
50
|
-
- ts-generics.md (strength: 0.28, type: learning)
|
|
51
|
-
- ts-utility-types.md (strength: 0.25, type: learning)
|
|
52
|
-
→ Proposed: Merge into "typescript-advanced-types.md" (strength: 0.65)
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
### 4. Get User Approval
|
|
56
|
-
|
|
57
|
-
Ask the user which groups to consolidate. They can:
|
|
58
|
-
- Approve all proposed groups
|
|
59
|
-
- Select specific groups
|
|
60
|
-
- Modify groupings
|
|
61
|
-
- Cancel
|
|
62
|
-
|
|
63
|
-
### 5. Perform Consolidation
|
|
64
|
-
|
|
65
|
-
For each approved group:
|
|
66
|
-
|
|
67
|
-
1. **Read all source memories** in the group
|
|
68
|
-
2. **Synthesize** a new consolidated memory that:
|
|
69
|
-
- Preserves all key details and insights from the sources
|
|
70
|
-
- Identifies patterns and evolution across the memories
|
|
71
|
-
- Creates a coherent narrative or structured summary
|
|
72
|
-
- Is more concise than the sum of its parts
|
|
73
|
-
3. **Calculate new strength**: `max(source_strengths) + 0.15` (capped at 1.0)
|
|
74
|
-
4. **Set new decay rate**: Use the slowest decay rate from the sources
|
|
75
|
-
5. **Merge tags**: Union of all source memory tags
|
|
76
|
-
6. **Set related**: Include all source memory IDs in the `related` field
|
|
77
|
-
7. **Salience anchoring**: The highest-salience memory in the group is the "anchor" — its framing and key details take priority in the synthesis. Inherit max salience.
|
|
78
|
-
8. **Confidence**: Inherit the maximum confidence from the group
|
|
79
|
-
9. **Cognitive type**: Set to `semantic` (consolidation abstracts knowledge)
|
|
80
|
-
|
|
81
|
-
### 6. Write Consolidated Memory
|
|
82
|
-
|
|
83
|
-
Create the new memory file in the appropriate location (same directory as the majority of sources, or in `_consolidated/` if sources span categories).
|
|
84
|
-
|
|
85
|
-
Use the standard memory format with an additional section:
|
|
86
|
-
|
|
87
|
-
```markdown
|
|
88
|
-
---
|
|
89
|
-
id: mem_<date>_<hex>
|
|
90
|
-
type: consolidated
|
|
91
|
-
cognitive_type: semantic
|
|
92
|
-
created: <now>
|
|
93
|
-
last_accessed: <now>
|
|
94
|
-
access_count: 0
|
|
95
|
-
recall_history: []
|
|
96
|
-
strength: <calculated>
|
|
97
|
-
decay_rate: <calculated>
|
|
98
|
-
salience: <max from sources>
|
|
99
|
-
confidence: <max from sources>
|
|
100
|
-
tags: [<merged tags>]
|
|
101
|
-
related: [<all source IDs>]
|
|
102
|
-
consolidated_from: [<source IDs>]
|
|
103
|
-
source: consolidation
|
|
104
|
-
encoding_context:
|
|
105
|
-
project: <project if shared, otherwise omit>
|
|
106
|
-
topics: [<merged key topics>]
|
|
107
|
-
task_type: <most common from sources>
|
|
108
|
-
---
|
|
109
|
-
|
|
110
|
-
# <Descriptive Title>
|
|
111
|
-
|
|
112
|
-
<Synthesized summary>
|
|
113
|
-
|
|
114
|
-
## Key Patterns
|
|
115
|
-
|
|
116
|
-
<Patterns identified across the source memories>
|
|
117
|
-
|
|
118
|
-
## Timeline
|
|
119
|
-
|
|
120
|
-
<Chronological evolution if applicable>
|
|
121
|
-
|
|
122
|
-
## Source Memories
|
|
123
|
-
|
|
124
|
-
<List of original memories that were merged, with their dates and original titles>
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
### 7. Archive Source Memories
|
|
128
|
-
|
|
129
|
-
Move the original source memory files to a `~/.brain/_archived/` directory (create if needed), preserving their paths as subdirectories. This keeps them recoverable and searchable via `/brain:remember`.
|
|
130
|
-
|
|
131
|
-
Update `index.json`:
|
|
132
|
-
- Remove archived memory entries from `memories`
|
|
133
|
-
- Add the new consolidated memory
|
|
134
|
-
- Update `memory_count`
|
|
135
|
-
- Update `last_updated`
|
|
136
|
-
|
|
137
|
-
Update `_archived/index.json`:
|
|
138
|
-
- Add entries for each archived memory with original metadata and `archived_reason: "consolidation"`
|
|
139
|
-
|
|
140
|
-
Update `associations.json`:
|
|
141
|
-
- Transfer association edges from source memories to the new consolidated memory
|
|
142
|
-
- Remove edges pointing to archived source memories
|
|
143
|
-
|
|
144
|
-
Update `review-queue.json`:
|
|
145
|
-
- Remove entries for archived source memories
|
|
146
|
-
- Add entry for the new consolidated memory if applicable
|
|
147
|
-
|
|
148
|
-
### 8. Confirm
|
|
149
|
-
|
|
150
|
-
Print a summary:
|
|
151
|
-
- How many memories were consolidated into how many
|
|
152
|
-
- New consolidated memory titles, paths, and strengths
|
|
153
|
-
- Space saved (number of files reduced)
|