brain-memory 0.1.0-beta.3 → 0.1.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +319 -0
- package/README.md +379 -95
- package/SECURITY.md +61 -0
- package/assets/brand/README.md +28 -0
- package/assets/brand/icon-dark.svg +19 -0
- package/assets/brand/icon-light.svg +20 -0
- package/assets/brand/linkedin_cover.png +0 -0
- package/assets/brand/linkedin_cover_light.png +0 -0
- package/assets/brand/mark-dark.svg +19 -0
- package/assets/brand/mark-light.svg +20 -0
- package/assets/brand/png/icon-dark-1024.png +0 -0
- package/assets/brand/png/icon-dark-128.png +0 -0
- package/assets/brand/png/icon-dark-256.png +0 -0
- package/assets/brand/png/icon-dark-512.png +0 -0
- package/assets/brand/png/icon-dark-64.png +0 -0
- package/assets/brand/png/icon-light-1024.png +0 -0
- package/assets/brand/png/icon-light-128.png +0 -0
- package/assets/brand/png/icon-light-256.png +0 -0
- package/assets/brand/png/icon-light-512.png +0 -0
- package/assets/brand/png/icon-light-64.png +0 -0
- package/assets/brand/png/mark-dark-1024.png +0 -0
- package/assets/brand/png/mark-light-1024.png +0 -0
- package/assets/icon.png +0 -0
- package/assets/icon.svg +19 -0
- package/bin/brain.js +111 -0
- package/bin/cloud-sync.js +150 -0
- package/bin/forget.js +122 -0
- package/bin/install.js +229 -254
- package/bin/memorize.js +458 -0
- package/bin/pin.js +44 -0
- package/bin/recall.js +251 -0
- package/bin/reinforce.js +187 -0
- package/bin/session-start.js +298 -0
- package/bin/skill.js +67 -0
- package/bin/unpin.js +38 -0
- package/commands/brain/forget.md +245 -15
- package/commands/brain/memorize.md +76 -144
- package/commands/brain/pin.md +79 -0
- package/commands/brain/remember.md +48 -105
- package/commands/brain/skills.md +60 -0
- package/commands/brain/sleep.md +69 -28
- package/commands/brain/status.md +24 -10
- package/commands/brain/sync.md +386 -0
- package/package.json +22 -8
- package/prompts/antigravity.md +180 -0
- package/prompts/claude.md +124 -20
- package/prompts/openai.md +124 -20
- package/prompts/opencode.md +179 -0
- package/src/cloud-sync.js +534 -0
- package/src/crypto.js +130 -0
- package/src/export-import.js +200 -0
- package/src/git-sync.js +444 -0
- package/src/index-manager.js +219 -18
- package/src/installer.js +444 -0
- package/src/pinning.js +124 -0
- package/src/scorer.js +22 -9
- package/src/skills.js +208 -0
- package/src/tfidf.js +501 -0
- package/commands/brain/consolidate.md +0 -153
- package/commands/brain/explore.md +0 -79
- package/commands/brain/init.md +0 -157
- package/commands/brain/review.md +0 -130
- package/commands/brain/sunshine.md +0 -337
- package/hooks/session-end.md +0 -48
- package/hooks/session-start.md +0 -48
- package/prompts/gemini.md +0 -76
package/src/skills.js
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Brain Memory — Procedural Skills (CoALA Phase 2 + Tier B §10.3)
|
|
3
|
+
*
|
|
4
|
+
* Stores "how to do things" as `_skills/<name>/SKILL.md` folders and serves them
|
|
5
|
+
* via three-level progressive disclosure:
|
|
6
|
+
* L0 — session start advertises only name + description (skills-index.json)
|
|
7
|
+
* L1 — on a matching task, the agent reads the full SKILL.md
|
|
8
|
+
* L2 — referenced resources/ load only at execution
|
|
9
|
+
*
|
|
10
|
+
* Procedural memory strengthens with successful use and weakens on failure
|
|
11
|
+
* (Tier B §10.3), so a bad crystallized skill demotes itself out of the index.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
16
|
+
|
|
17
|
+
const {
|
|
18
|
+
getBrainDir,
|
|
19
|
+
readSkillsIndex,
|
|
20
|
+
writeSkillsIndex,
|
|
21
|
+
validateBrainPath,
|
|
22
|
+
atomicWriteSync,
|
|
23
|
+
} = require('./index-manager');
|
|
24
|
+
const { setFrontmatterFields } = require('./pinning');
|
|
25
|
+
|
|
26
|
+
const SKILLS_DIR = '_skills';
|
|
27
|
+
const DEFAULT_STRENGTH = 0.6;
|
|
28
|
+
// A skill whose failure rate exceeds this drops below the L0 advertisement cut.
|
|
29
|
+
const DEMOTE_FAIL_RATIO = 0.5;
|
|
30
|
+
|
|
31
|
+
/** Normalize a skill name to a filesystem-safe slug. */
|
|
32
|
+
function slug(name) {
|
|
33
|
+
return String(name || '').toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function skillPath(name) {
|
|
37
|
+
return `${SKILLS_DIR}/${slug(name)}/SKILL.md`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function buildSkillFile(skill) {
|
|
41
|
+
const fm = [
|
|
42
|
+
'---',
|
|
43
|
+
`name: ${slug(skill.name)}`,
|
|
44
|
+
`description: ${JSON.stringify(skill.description || '')}`,
|
|
45
|
+
`triggers: [${(skill.triggers || []).map((t) => JSON.stringify(t)).join(', ')}]`,
|
|
46
|
+
'cognitive_type: procedural',
|
|
47
|
+
`strength: ${skill.strength ?? DEFAULT_STRENGTH}`,
|
|
48
|
+
`last_used: ${skill.last_used ? JSON.stringify(skill.last_used) : 'null'}`,
|
|
49
|
+
`use_count: ${skill.use_count ?? 0}`,
|
|
50
|
+
`fail_count: ${skill.fail_count ?? 0}`,
|
|
51
|
+
'---',
|
|
52
|
+
'',
|
|
53
|
+
].join('\n');
|
|
54
|
+
return fm + (skill.body || '') + '\n';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Is this skill advertised at L0? Demoted skills stay on disk but go quiet. */
|
|
58
|
+
function isAdvertised(s) {
|
|
59
|
+
const uses = s.use_count || 0;
|
|
60
|
+
const fails = s.fail_count || 0;
|
|
61
|
+
return !(uses >= 3 && fails / uses > DEMOTE_FAIL_RATIO);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Create or replace a skill.
|
|
66
|
+
* @param {string} [projectRoot]
|
|
67
|
+
* @param {Object} skill - { name, description, triggers[], body, strength? }
|
|
68
|
+
*/
|
|
69
|
+
function addSkill(projectRoot, skill) {
|
|
70
|
+
const name = slug(skill && skill.name);
|
|
71
|
+
if (!name) return { error: 'Skill name required' };
|
|
72
|
+
|
|
73
|
+
const brainDir = getBrainDir(projectRoot);
|
|
74
|
+
const dir = path.join(brainDir, SKILLS_DIR, name);
|
|
75
|
+
validateBrainPath(dir, brainDir);
|
|
76
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
77
|
+
|
|
78
|
+
const record = {
|
|
79
|
+
name,
|
|
80
|
+
description: skill.description || '',
|
|
81
|
+
triggers: skill.triggers || [],
|
|
82
|
+
strength: skill.strength ?? DEFAULT_STRENGTH,
|
|
83
|
+
use_count: 0,
|
|
84
|
+
fail_count: 0,
|
|
85
|
+
last_used: null,
|
|
86
|
+
};
|
|
87
|
+
atomicWriteSync(path.join(dir, 'SKILL.md'), buildSkillFile({ ...record, body: skill.body }));
|
|
88
|
+
|
|
89
|
+
const idx = readSkillsIndex(projectRoot);
|
|
90
|
+
idx.skills = idx.skills.filter((s) => s.name !== name);
|
|
91
|
+
idx.skills.push({ ...record, path: skillPath(name) });
|
|
92
|
+
writeSkillsIndex(idx, projectRoot);
|
|
93
|
+
|
|
94
|
+
return { name, added: true };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Full advertised index (every stored skill, advertised or demoted). */
|
|
98
|
+
function listSkills(projectRoot) {
|
|
99
|
+
return readSkillsIndex(projectRoot).skills;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** L0 summaries (name + description) for advertised skills only. */
|
|
103
|
+
function advertisedSummaries(projectRoot) {
|
|
104
|
+
return readSkillsIndex(projectRoot).skills
|
|
105
|
+
.filter(isAdvertised)
|
|
106
|
+
.map((s) => ({ name: s.name, description: s.description, triggers: s.triggers || [] }));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** L1: the full SKILL.md body. */
|
|
110
|
+
function showSkill(projectRoot, name) {
|
|
111
|
+
const full = path.join(getBrainDir(projectRoot), skillPath(name));
|
|
112
|
+
try {
|
|
113
|
+
return { name: slug(name), content: fs.readFileSync(full, 'utf-8') };
|
|
114
|
+
} catch (_) {
|
|
115
|
+
return { error: `Skill not found: ${name}` };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Record a use. Success strengthens; failure weakens and counts toward demotion.
|
|
121
|
+
* @param {Object} [opts] - { failed: boolean }
|
|
122
|
+
*/
|
|
123
|
+
function useSkill(projectRoot, name, opts = {}) {
|
|
124
|
+
const idx = readSkillsIndex(projectRoot);
|
|
125
|
+
const s = idx.skills.find((x) => x.name === slug(name));
|
|
126
|
+
if (!s) return { error: `Skill not found: ${name}` };
|
|
127
|
+
|
|
128
|
+
s.use_count = (s.use_count || 0) + 1;
|
|
129
|
+
if (opts.failed) {
|
|
130
|
+
s.fail_count = (s.fail_count || 0) + 1;
|
|
131
|
+
s.strength = Math.max(0, (s.strength ?? DEFAULT_STRENGTH) - 0.10);
|
|
132
|
+
} else {
|
|
133
|
+
const cur = s.strength ?? DEFAULT_STRENGTH;
|
|
134
|
+
s.strength = Math.min(1.0, cur + 0.05 * (1.0 - cur));
|
|
135
|
+
}
|
|
136
|
+
s.last_used = new Date().toISOString();
|
|
137
|
+
writeSkillsIndex(idx, projectRoot);
|
|
138
|
+
|
|
139
|
+
setFrontmatterFields(getBrainDir(projectRoot), skillPath(name), {
|
|
140
|
+
strength: Math.round(s.strength * 1000) / 1000,
|
|
141
|
+
use_count: s.use_count,
|
|
142
|
+
fail_count: s.fail_count,
|
|
143
|
+
last_used: s.last_used,
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
name: s.name,
|
|
148
|
+
use_count: s.use_count,
|
|
149
|
+
fail_count: s.fail_count,
|
|
150
|
+
strength: Math.round(s.strength * 1000) / 1000,
|
|
151
|
+
advertised: isAdvertised(s),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Export a stored skill into the host agent's native skill format so it becomes
|
|
157
|
+
* directly executable (CoALA Phase 4 host bridge). Brain stores/distills; the
|
|
158
|
+
* host executes.
|
|
159
|
+
*
|
|
160
|
+
* @param {string} [projectRoot] - Brain filesystem root
|
|
161
|
+
* @param {string} name - Skill name
|
|
162
|
+
* @param {string} [target='claude'] - 'claude' | 'gemini'
|
|
163
|
+
* @param {string} [destRoot=process.cwd()] - Host project root to write into
|
|
164
|
+
*/
|
|
165
|
+
function exportSkill(projectRoot, name, target = 'claude', destRoot = process.cwd()) {
|
|
166
|
+
const src = showSkill(projectRoot, name);
|
|
167
|
+
if (src.error) return src;
|
|
168
|
+
|
|
169
|
+
const dests = {
|
|
170
|
+
claude: path.join(destRoot, '.claude', 'skills', slug(name)),
|
|
171
|
+
gemini: path.join(destRoot, '.gemini', 'skills', slug(name)),
|
|
172
|
+
};
|
|
173
|
+
const dir = dests[target];
|
|
174
|
+
if (!dir) return { error: `Unknown target: ${target} (use claude|gemini)` };
|
|
175
|
+
|
|
176
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
177
|
+
const outPath = path.join(dir, 'SKILL.md');
|
|
178
|
+
fs.writeFileSync(outPath, src.content);
|
|
179
|
+
return { name: slug(name), target, path: outPath };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function removeSkill(projectRoot, name) {
|
|
183
|
+
const n = slug(name);
|
|
184
|
+
const dir = path.join(getBrainDir(projectRoot), SKILLS_DIR, n);
|
|
185
|
+
let removedDir = false;
|
|
186
|
+
try { fs.rmSync(dir, { recursive: true, force: true }); removedDir = true; } catch (_) { /* ignore */ }
|
|
187
|
+
|
|
188
|
+
const idx = readSkillsIndex(projectRoot);
|
|
189
|
+
const before = idx.skills.length;
|
|
190
|
+
idx.skills = idx.skills.filter((s) => s.name !== n);
|
|
191
|
+
writeSkillsIndex(idx, projectRoot);
|
|
192
|
+
|
|
193
|
+
return { name: n, removed: removedDir || before !== idx.skills.length };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
module.exports = {
|
|
197
|
+
slug,
|
|
198
|
+
skillPath,
|
|
199
|
+
isAdvertised,
|
|
200
|
+
addSkill,
|
|
201
|
+
listSkills,
|
|
202
|
+
advertisedSummaries,
|
|
203
|
+
showSkill,
|
|
204
|
+
useSkill,
|
|
205
|
+
removeSkill,
|
|
206
|
+
exportSkill,
|
|
207
|
+
DEMOTE_FAIL_RATIO,
|
|
208
|
+
};
|
package/src/tfidf.js
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Brain Memory — TF-IDF Search Engine
|
|
3
|
+
*
|
|
4
|
+
* Lightweight, zero-dependency TF-IDF implementation for semantic search
|
|
5
|
+
* across brain memories. Builds a search index at memorize time and
|
|
6
|
+
* computes cosine similarity at recall time.
|
|
7
|
+
*
|
|
8
|
+
* This replaces the "agent eyeballs relevance" approach with deterministic
|
|
9
|
+
* math: tokenize → TF-IDF weights → cosine similarity → numeric score.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
|
|
15
|
+
const SEARCH_INDEX_FILE = 'search-index.json';
|
|
16
|
+
|
|
17
|
+
// Common English stopwords — filtered out during tokenization
|
|
18
|
+
const STOPWORDS = new Set([
|
|
19
|
+
'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
|
20
|
+
'of', 'with', 'by', 'from', 'is', 'was', 'are', 'were', 'be', 'been',
|
|
21
|
+
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
|
22
|
+
'could', 'should', 'may', 'might', 'shall', 'can', 'need', 'must',
|
|
23
|
+
'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'we',
|
|
24
|
+
'our', 'you', 'your', 'he', 'him', 'his', 'she', 'her', 'they', 'them',
|
|
25
|
+
'their', 'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how',
|
|
26
|
+
'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
|
|
27
|
+
'such', 'no', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
|
|
28
|
+
'just', 'about', 'above', 'after', 'again', 'also', 'any', 'because',
|
|
29
|
+
'before', 'between', 'during', 'if', 'into', 'then', 'there', 'up',
|
|
30
|
+
'out', 'over', 'under', 'here', 'as', 'like', 'use', 'used', 'using',
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Simple suffix stemmer — strips common English suffixes.
|
|
35
|
+
* Not as thorough as Porter/Snowball but zero-dependency and good enough
|
|
36
|
+
* for matching "connection" → "connect", "pooling" → "pool", etc.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} word - Lowercase word
|
|
39
|
+
* @returns {string} Stemmed word
|
|
40
|
+
*/
|
|
41
|
+
function stem(word) {
|
|
42
|
+
if (word.length <= 3) return word;
|
|
43
|
+
|
|
44
|
+
// Two-pass stemmer: strip compound suffixes first, then simple ones.
|
|
45
|
+
// Pass 1: long compound suffixes
|
|
46
|
+
const longSuffixes = [
|
|
47
|
+
'izations', 'ational', 'fulness', 'ousness', 'iveness',
|
|
48
|
+
'ization', 'isation', 'tional', 'ations', 'ments',
|
|
49
|
+
'ation', 'ness', 'ment', 'ence', 'ance', 'ible', 'able',
|
|
50
|
+
'ally', 'ious', 'eous', 'ings', 'ions',
|
|
51
|
+
'ity', 'ful', 'ous', 'ive', 'ize', 'ise', 'ion',
|
|
52
|
+
'ing', 'ies',
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
for (const suffix of longSuffixes) {
|
|
56
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
|
|
57
|
+
return word.slice(0, -suffix.length);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Pass 2: short suffixes (conservative — skip ambiguous word endings)
|
|
62
|
+
if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2);
|
|
63
|
+
if (word.endsWith('er') && word.length > 4) return word.slice(0, -2);
|
|
64
|
+
if (word.endsWith('ly') && word.length > 4) return word.slice(0, -2);
|
|
65
|
+
if (word.endsWith('al') && word.length > 4) return word.slice(0, -2);
|
|
66
|
+
if (word.endsWith('es') && word.length > 4) return word.slice(0, -2);
|
|
67
|
+
|
|
68
|
+
// Strip trailing 's' only for likely plurals (not ss, us, is endings)
|
|
69
|
+
if (word.endsWith('s') && !word.endsWith('ss') && !word.endsWith('us') &&
|
|
70
|
+
!word.endsWith('is') && word.length > 4) {
|
|
71
|
+
return word.slice(0, -1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return word;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Tokenize text into stemmed terms, filtering stopwords.
|
|
79
|
+
*
|
|
80
|
+
* @param {string} text - Input text
|
|
81
|
+
* @returns {string[]} Array of stemmed tokens
|
|
82
|
+
*/
|
|
83
|
+
function tokenize(text) {
|
|
84
|
+
if (!text) return [];
|
|
85
|
+
|
|
86
|
+
return text
|
|
87
|
+
.toLowerCase()
|
|
88
|
+
.replace(/[^a-z0-9\s_-]/g, ' ') // strip punctuation
|
|
89
|
+
.split(/\s+/) // split on whitespace
|
|
90
|
+
.filter((w) => w.length >= 2 && !STOPWORDS.has(w))
|
|
91
|
+
.map(stem);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Compute term frequencies for a token list.
|
|
96
|
+
*
|
|
97
|
+
* @param {string[]} tokens - Array of tokens
|
|
98
|
+
* @returns {Object} Map of term → count
|
|
99
|
+
*/
|
|
100
|
+
function termFrequencies(tokens) {
|
|
101
|
+
const tf = {};
|
|
102
|
+
for (const token of tokens) {
|
|
103
|
+
tf[token] = (tf[token] || 0) + 1;
|
|
104
|
+
}
|
|
105
|
+
return tf;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Compute TF-IDF vector for a document given corpus-wide IDF values.
|
|
110
|
+
*
|
|
111
|
+
* TF = (term count in doc) / (total terms in doc)
|
|
112
|
+
* IDF = log(N / df) where df = docs containing the term
|
|
113
|
+
*
|
|
114
|
+
* @param {Object} tf - Term frequency map for this document
|
|
115
|
+
* @param {Object} idf - IDF values (term → idf weight)
|
|
116
|
+
* @param {number} docLength - Total number of tokens in document
|
|
117
|
+
* @returns {Object} Sparse TF-IDF vector (term → weight)
|
|
118
|
+
*/
|
|
119
|
+
function computeTfidfVector(tf, idf, docLength) {
|
|
120
|
+
const vector = {};
|
|
121
|
+
if (docLength === 0) return vector;
|
|
122
|
+
|
|
123
|
+
for (const [term, count] of Object.entries(tf)) {
|
|
124
|
+
const tfNorm = count / docLength;
|
|
125
|
+
const idfVal = idf[term] || 0;
|
|
126
|
+
const weight = tfNorm * idfVal;
|
|
127
|
+
if (weight > 0) {
|
|
128
|
+
vector[term] = Math.round(weight * 10000) / 10000;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return vector;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Cosine similarity between two sparse vectors.
|
|
136
|
+
*
|
|
137
|
+
* @param {Object} a - Sparse vector (term → weight)
|
|
138
|
+
* @param {Object} b - Sparse vector (term → weight)
|
|
139
|
+
* @returns {number} Similarity (0.0-1.0)
|
|
140
|
+
*/
|
|
141
|
+
function cosineSimilarity(a, b) {
|
|
142
|
+
let dot = 0;
|
|
143
|
+
let magA = 0;
|
|
144
|
+
let magB = 0;
|
|
145
|
+
|
|
146
|
+
for (const [term, weight] of Object.entries(a)) {
|
|
147
|
+
magA += weight * weight;
|
|
148
|
+
if (b[term]) {
|
|
149
|
+
dot += weight * b[term];
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
for (const weight of Object.values(b)) {
|
|
154
|
+
magB += weight * weight;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
158
|
+
return denom > 0 ? dot / denom : 0;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Build text content from a memory for indexing.
|
|
163
|
+
* Concatenates title, body, and tags with appropriate weighting
|
|
164
|
+
* (title terms appear 3x, tags 2x, body 1x).
|
|
165
|
+
*
|
|
166
|
+
* @param {Object} memory - Memory object with title, body, tags
|
|
167
|
+
* @returns {string} Combined text for tokenization
|
|
168
|
+
*/
|
|
169
|
+
function buildMemoryText(memory) {
|
|
170
|
+
const parts = [];
|
|
171
|
+
|
|
172
|
+
// Title gets 3x weight (repeat it)
|
|
173
|
+
if (memory.title) {
|
|
174
|
+
parts.push(memory.title, memory.title, memory.title);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Tags get 2x weight
|
|
178
|
+
if (memory.tags && Array.isArray(memory.tags)) {
|
|
179
|
+
const tagText = memory.tags.join(' ');
|
|
180
|
+
parts.push(tagText, tagText);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Body gets 1x weight
|
|
184
|
+
if (memory.body) {
|
|
185
|
+
parts.push(memory.body);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Content field (used in some memories)
|
|
189
|
+
if (memory.content && memory.content !== memory.body) {
|
|
190
|
+
parts.push(memory.content);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return parts.join(' ');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ─────────────────────────────────────────────────────────
|
|
197
|
+
// Search Index Management
|
|
198
|
+
// ─────────────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Read the search index from disk.
|
|
202
|
+
*
|
|
203
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
204
|
+
* @returns {Object|null} Search index or null if not found
|
|
205
|
+
*/
|
|
206
|
+
function readSearchIndex(brainDir) {
|
|
207
|
+
const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
|
|
208
|
+
try {
|
|
209
|
+
return JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
|
|
210
|
+
} catch (err) {
|
|
211
|
+
if (err.code === 'ENOENT') return null;
|
|
212
|
+
throw err;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Write the search index to disk (atomically).
|
|
218
|
+
*
|
|
219
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
220
|
+
* @param {Object} searchIndex - The search index object
|
|
221
|
+
*/
|
|
222
|
+
function writeSearchIndex(brainDir, searchIndex) {
|
|
223
|
+
const indexPath = path.join(brainDir, SEARCH_INDEX_FILE);
|
|
224
|
+
const crypto = require('crypto');
|
|
225
|
+
const tmpPath = indexPath + '.tmp.' + crypto.randomBytes(4).toString('hex');
|
|
226
|
+
fs.writeFileSync(tmpPath, JSON.stringify(searchIndex, null, 2) + '\n');
|
|
227
|
+
fs.renameSync(tmpPath, indexPath);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Create a new empty search index.
|
|
232
|
+
*
|
|
233
|
+
* @returns {Object} Empty search index
|
|
234
|
+
*/
|
|
235
|
+
function createSearchIndex() {
|
|
236
|
+
return {
|
|
237
|
+
version: 1,
|
|
238
|
+
doc_count: 0,
|
|
239
|
+
documents: {}, // memoryId → { tf: {term→count}, length: N }
|
|
240
|
+
df: {}, // term → number of documents containing it
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Recompute IDF values from the current document frequencies.
|
|
246
|
+
*
|
|
247
|
+
* @param {Object} searchIndex - The search index
|
|
248
|
+
* @returns {Object} IDF map (term → idf weight)
|
|
249
|
+
*/
|
|
250
|
+
function computeIdf(searchIndex) {
|
|
251
|
+
const N = searchIndex.doc_count;
|
|
252
|
+
if (N === 0) return {};
|
|
253
|
+
|
|
254
|
+
const idf = {};
|
|
255
|
+
for (const [term, df] of Object.entries(searchIndex.df)) {
|
|
256
|
+
// Smoothed IDF: ensures terms present in all docs still get a small positive weight
|
|
257
|
+
idf[term] = Math.log(1 + N / df);
|
|
258
|
+
}
|
|
259
|
+
return idf;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Add a memory document to the search index.
|
|
264
|
+
*
|
|
265
|
+
* @param {Object} searchIndex - The search index (mutated)
|
|
266
|
+
* @param {string} memoryId - Memory ID
|
|
267
|
+
* @param {Object} memory - Memory object (title, body, tags, content)
|
|
268
|
+
* @returns {Object} The mutated search index
|
|
269
|
+
*/
|
|
270
|
+
function addDocument(searchIndex, memoryId, memory) {
|
|
271
|
+
// Remove old version if re-indexing
|
|
272
|
+
if (searchIndex.documents[memoryId]) {
|
|
273
|
+
removeDocument(searchIndex, memoryId);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const text = buildMemoryText(memory);
|
|
277
|
+
const tokens = tokenize(text);
|
|
278
|
+
const tf = termFrequencies(tokens);
|
|
279
|
+
|
|
280
|
+
searchIndex.documents[memoryId] = {
|
|
281
|
+
tf,
|
|
282
|
+
length: tokens.length,
|
|
283
|
+
};
|
|
284
|
+
searchIndex.doc_count++;
|
|
285
|
+
|
|
286
|
+
// Update document frequencies
|
|
287
|
+
for (const term of Object.keys(tf)) {
|
|
288
|
+
searchIndex.df[term] = (searchIndex.df[term] || 0) + 1;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return searchIndex;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Remove a memory document from the search index.
|
|
296
|
+
*
|
|
297
|
+
* @param {Object} searchIndex - The search index (mutated)
|
|
298
|
+
* @param {string} memoryId - Memory ID
|
|
299
|
+
* @returns {Object} The mutated search index
|
|
300
|
+
*/
|
|
301
|
+
function removeDocument(searchIndex, memoryId) {
|
|
302
|
+
const doc = searchIndex.documents[memoryId];
|
|
303
|
+
if (!doc) return searchIndex;
|
|
304
|
+
|
|
305
|
+
// Decrement document frequencies
|
|
306
|
+
for (const term of Object.keys(doc.tf)) {
|
|
307
|
+
if (searchIndex.df[term]) {
|
|
308
|
+
searchIndex.df[term]--;
|
|
309
|
+
if (searchIndex.df[term] <= 0) {
|
|
310
|
+
delete searchIndex.df[term];
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
delete searchIndex.documents[memoryId];
|
|
316
|
+
searchIndex.doc_count--;
|
|
317
|
+
|
|
318
|
+
return searchIndex;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Compute TF-IDF relevance scores for a query against all indexed memories.
|
|
323
|
+
*
|
|
324
|
+
* @param {Object} searchIndex - The search index
|
|
325
|
+
* @param {string} query - The search query
|
|
326
|
+
* @returns {Object} Map of memoryId → relevance score (0.0-1.0)
|
|
327
|
+
*/
|
|
328
|
+
function search(searchIndex, query) {
|
|
329
|
+
if (!searchIndex || searchIndex.doc_count === 0) return {};
|
|
330
|
+
|
|
331
|
+
const queryTokens = tokenize(query);
|
|
332
|
+
if (queryTokens.length === 0) return {};
|
|
333
|
+
|
|
334
|
+
const queryTf = termFrequencies(queryTokens);
|
|
335
|
+
const idf = computeIdf(searchIndex);
|
|
336
|
+
const queryVector = computeTfidfVector(queryTf, idf, queryTokens.length);
|
|
337
|
+
|
|
338
|
+
const scores = {};
|
|
339
|
+
|
|
340
|
+
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
341
|
+
const docVector = computeTfidfVector(doc.tf, idf, doc.length);
|
|
342
|
+
const sim = cosineSimilarity(queryVector, docVector);
|
|
343
|
+
if (sim > 0) {
|
|
344
|
+
scores[memoryId] = Math.round(sim * 1000) / 1000;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return scores;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* BM25 relevance scoring for a query against all indexed memories.
|
|
353
|
+
*
|
|
354
|
+
* Fixes two TF-IDF-cosine weaknesses that buried long, detailed memories under
|
|
355
|
+
* short distractors: BM25 SATURATES term frequency (k1) and normalizes document
|
|
356
|
+
* length GENTLY (b), instead of dividing TF by full length and cosine by full
|
|
357
|
+
* magnitude (which penalized the richer oracle memories for being longer). The
|
|
358
|
+
* field weighting baked into the index (title 3x, tags 2x via buildMemoryText)
|
|
359
|
+
* is preserved. Scores are normalized to 0..1 by the top score so they blend
|
|
360
|
+
* cleanly into the scorer's recall formula (which expects relevance in [0,1]).
|
|
361
|
+
*
|
|
362
|
+
* @param {Object} searchIndex - The search index
|
|
363
|
+
* @param {string} query - The search query
|
|
364
|
+
* @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
|
|
365
|
+
* @returns {Object} Map of memoryId → normalized relevance score (0.0-1.0)
|
|
366
|
+
*/
|
|
367
|
+
function bm25Search(searchIndex, query, opts = {}) {
|
|
368
|
+
if (!searchIndex || searchIndex.doc_count === 0) return {};
|
|
369
|
+
|
|
370
|
+
const queryTokens = tokenize(query);
|
|
371
|
+
if (queryTokens.length === 0) return {};
|
|
372
|
+
|
|
373
|
+
const k1 = opts.k1 != null ? opts.k1 : 1.5;
|
|
374
|
+
const b = opts.b != null ? opts.b : 0.75;
|
|
375
|
+
const N = searchIndex.doc_count;
|
|
376
|
+
|
|
377
|
+
// Average document length (for length normalization).
|
|
378
|
+
let totalLen = 0;
|
|
379
|
+
for (const d of Object.values(searchIndex.documents)) totalLen += d.length || 0;
|
|
380
|
+
const avgdl = N > 0 ? totalLen / N : 0;
|
|
381
|
+
|
|
382
|
+
// BM25 IDF per (deduped) query term, floored at 0 so a term present in nearly
|
|
383
|
+
// every document can't contribute a negative score.
|
|
384
|
+
const qTerms = [...new Set(queryTokens)];
|
|
385
|
+
const idf = {};
|
|
386
|
+
for (const t of qTerms) {
|
|
387
|
+
const df = searchIndex.df[t] || 0;
|
|
388
|
+
idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const raw = {};
|
|
392
|
+
let maxScore = 0;
|
|
393
|
+
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
394
|
+
const dl = doc.length || 0;
|
|
395
|
+
let score = 0;
|
|
396
|
+
for (const t of qTerms) {
|
|
397
|
+
const f = doc.tf[t] || 0;
|
|
398
|
+
if (f === 0) continue;
|
|
399
|
+
const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
|
|
400
|
+
score += idf[t] * (f * (k1 + 1)) / denom;
|
|
401
|
+
}
|
|
402
|
+
if (score > 0) {
|
|
403
|
+
raw[memoryId] = score;
|
|
404
|
+
if (score > maxScore) maxScore = score;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Normalize to 0..1 by the top score — preserves ranking and keeps the [0,1]
|
|
409
|
+
// contract the recall scorer's relevance blend expects.
|
|
410
|
+
const scores = {};
|
|
411
|
+
if (maxScore > 0) {
|
|
412
|
+
for (const [id, s] of Object.entries(raw)) {
|
|
413
|
+
scores[id] = Math.round((s / maxScore) * 1000) / 1000;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
return scores;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Rebuild the entire search index from memory files on disk.
|
|
421
|
+
*
|
|
422
|
+
* @param {string} brainDir - Absolute path to ~/.brain/
|
|
423
|
+
* @param {Object} index - The brain index (index.json)
|
|
424
|
+
* @returns {Object} Fresh search index
|
|
425
|
+
*/
|
|
426
|
+
function rebuildIndex(brainDir, index) {
|
|
427
|
+
const searchIndex = createSearchIndex();
|
|
428
|
+
|
|
429
|
+
if (!index || !index.memories) return searchIndex;
|
|
430
|
+
|
|
431
|
+
for (const [memoryId, entry] of Object.entries(index.memories)) {
|
|
432
|
+
// Read the actual memory file for full content
|
|
433
|
+
const memPath = path.join(brainDir, entry.path);
|
|
434
|
+
let body = '';
|
|
435
|
+
let content = '';
|
|
436
|
+
try {
|
|
437
|
+
const raw = fs.readFileSync(memPath, 'utf-8');
|
|
438
|
+
// Extract body from markdown (after frontmatter)
|
|
439
|
+
const fmEnd = raw.indexOf('---', raw.indexOf('---') + 3);
|
|
440
|
+
if (fmEnd !== -1) {
|
|
441
|
+
body = raw.slice(fmEnd + 3).trim();
|
|
442
|
+
}
|
|
443
|
+
} catch { /* skip unreadable files */ }
|
|
444
|
+
|
|
445
|
+
addDocument(searchIndex, memoryId, {
|
|
446
|
+
title: entry.title || '',
|
|
447
|
+
body,
|
|
448
|
+
tags: entry.tags || [],
|
|
449
|
+
content,
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
return searchIndex;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Is the persisted search index out of sync with index.json?
|
|
458
|
+
*
|
|
459
|
+
* The search index drifts whenever memories change outside memorize's
|
|
460
|
+
* addDocument path (forget, sleep/consolidate, sync pull, manual edit) — and a
|
|
461
|
+
* present-but-empty index silently makes every relevance score 0. Both `recall`
|
|
462
|
+
* and `session-start` must rebuild when the doc SET (not mere presence) drifts.
|
|
463
|
+
*
|
|
464
|
+
* @param {Object|null} searchIndex - parsed search-index.json (or null if absent)
|
|
465
|
+
* @param {Object} index - parsed index.json
|
|
466
|
+
* @returns {boolean} true when a rebuild is required
|
|
467
|
+
*/
|
|
468
|
+
function isSearchIndexStale(searchIndex, index) {
|
|
469
|
+
if (!searchIndex || !searchIndex.documents) return true;
|
|
470
|
+
const memIds = Object.keys((index && index.memories) || {});
|
|
471
|
+
const docCount = searchIndex.doc_count != null
|
|
472
|
+
? searchIndex.doc_count
|
|
473
|
+
: Object.keys(searchIndex.documents).length;
|
|
474
|
+
if (docCount !== memIds.length) return true;
|
|
475
|
+
for (const id of memIds) if (!(id in searchIndex.documents)) return true;
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
module.exports = {
|
|
480
|
+
// Core primitives
|
|
481
|
+
tokenize,
|
|
482
|
+
stem,
|
|
483
|
+
termFrequencies,
|
|
484
|
+
computeTfidfVector,
|
|
485
|
+
cosineSimilarity,
|
|
486
|
+
buildMemoryText,
|
|
487
|
+
// Index management
|
|
488
|
+
createSearchIndex,
|
|
489
|
+
readSearchIndex,
|
|
490
|
+
writeSearchIndex,
|
|
491
|
+
addDocument,
|
|
492
|
+
removeDocument,
|
|
493
|
+
computeIdf,
|
|
494
|
+
search,
|
|
495
|
+
bm25Search,
|
|
496
|
+
rebuildIndex,
|
|
497
|
+
isSearchIndexStale,
|
|
498
|
+
// Constants
|
|
499
|
+
SEARCH_INDEX_FILE,
|
|
500
|
+
STOPWORDS,
|
|
501
|
+
};
|