brain-memory 0.1.0-beta.26 → 0.1.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +275 -0
- package/README.md +90 -21
- package/SECURITY.md +61 -0
- package/assets/icon.png +0 -0
- package/assets/icon.svg +19 -1
- package/bin/forget.js +120 -0
- package/bin/install.js +15 -13
- package/bin/memorize.js +6 -1
- package/bin/recall.js +17 -3
- package/bin/session-start.js +8 -2
- package/package.json +3 -1
- package/prompts/{gemini.md → antigravity.md} +18 -18
- package/prompts/openai.md +8 -8
- package/src/installer.js +73 -21
- package/src/scorer.js +14 -7
- package/src/tfidf.js +93 -0
package/src/scorer.js
CHANGED
|
@@ -365,16 +365,21 @@ function computeSpreadingActivationBatch(scoredMemories, associations, maxDepth
|
|
|
365
365
|
* @returns {Object[]} Scored and sorted memories (highest first)
|
|
366
366
|
*/
|
|
367
367
|
function rankMemories(memories, relevanceFn, options = {}) {
|
|
368
|
+
// Guard every term against NaN (e.g. a missing/invalid last_accessed or
|
|
369
|
+
// strength) — a single NaN score sorts non-deterministically and can outrank
|
|
370
|
+
// valid results.
|
|
371
|
+
const fin = (x) => (Number.isFinite(x) ? x : 0);
|
|
372
|
+
|
|
368
373
|
// First pass: compute base scores
|
|
369
374
|
const scored = memories.map((mem) => {
|
|
370
|
-
const decayedStrength = computeDecayedStrength(
|
|
375
|
+
const decayedStrength = fin(computeDecayedStrength(
|
|
371
376
|
mem.strength,
|
|
372
377
|
mem.decay_rate,
|
|
373
378
|
mem.last_accessed,
|
|
374
379
|
!!(mem.stable || mem.pinned) // pinned ⇒ implicitly stable (decay-exempt)
|
|
375
|
-
);
|
|
376
|
-
const recencyBonus = computeRecencyBonus(mem.last_accessed);
|
|
377
|
-
const relevance = relevanceFn(mem);
|
|
380
|
+
));
|
|
381
|
+
const recencyBonus = fin(computeRecencyBonus(mem.last_accessed));
|
|
382
|
+
const relevance = fin(relevanceFn(mem));
|
|
378
383
|
|
|
379
384
|
return {
|
|
380
385
|
...mem,
|
|
@@ -420,12 +425,12 @@ function rankMemories(memories, relevanceFn, options = {}) {
|
|
|
420
425
|
if (mem.context_match > 0) extras.contextMatch = mem.context_match;
|
|
421
426
|
if (mem.salience != null) extras.salience = mem.salience;
|
|
422
427
|
|
|
423
|
-
const score = computeRecallScore(
|
|
428
|
+
const score = fin(computeRecallScore(
|
|
424
429
|
mem.relevance,
|
|
425
430
|
mem.decayed_strength,
|
|
426
431
|
mem.recency_bonus,
|
|
427
432
|
extras
|
|
428
|
-
);
|
|
433
|
+
));
|
|
429
434
|
|
|
430
435
|
return {
|
|
431
436
|
...mem,
|
|
@@ -437,7 +442,9 @@ function rankMemories(memories, relevanceFn, options = {}) {
|
|
|
437
442
|
score: Math.round(score * 1000) / 1000,
|
|
438
443
|
};
|
|
439
444
|
})
|
|
440
|
-
|
|
445
|
+
// Deterministic order: score desc, then id asc to break ties (scores are
|
|
446
|
+
// rounded to 3dp, which manufactures ties at scale).
|
|
447
|
+
.sort((a, b) => (b.score - a.score) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
441
448
|
}
|
|
442
449
|
|
|
443
450
|
module.exports = {
|
package/src/tfidf.js
CHANGED
|
@@ -348,6 +348,74 @@ function search(searchIndex, query) {
|
|
|
348
348
|
return scores;
|
|
349
349
|
}
|
|
350
350
|
|
|
351
|
+
/**
|
|
352
|
+
* BM25 relevance scoring for a query against all indexed memories.
|
|
353
|
+
*
|
|
354
|
+
* Fixes two TF-IDF-cosine weaknesses that buried long, detailed memories under
|
|
355
|
+
* short distractors: BM25 SATURATES term frequency (k1) and normalizes document
|
|
356
|
+
* length GENTLY (b), instead of dividing TF by full length and cosine by full
|
|
357
|
+
* magnitude (which penalized the richer oracle memories for being longer). The
|
|
358
|
+
* field weighting baked into the index (title 3x, tags 2x via buildMemoryText)
|
|
359
|
+
* is preserved. Scores are normalized to 0..1 by the top score so they blend
|
|
360
|
+
* cleanly into the scorer's recall formula (which expects relevance in [0,1]).
|
|
361
|
+
*
|
|
362
|
+
* @param {Object} searchIndex - The search index
|
|
363
|
+
* @param {string} query - The search query
|
|
364
|
+
* @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
|
|
365
|
+
* @returns {Object} Map of memoryId → normalized relevance score (0.0-1.0)
|
|
366
|
+
*/
|
|
367
|
+
function bm25Search(searchIndex, query, opts = {}) {
|
|
368
|
+
if (!searchIndex || searchIndex.doc_count === 0) return {};
|
|
369
|
+
|
|
370
|
+
const queryTokens = tokenize(query);
|
|
371
|
+
if (queryTokens.length === 0) return {};
|
|
372
|
+
|
|
373
|
+
const k1 = opts.k1 != null ? opts.k1 : 1.5;
|
|
374
|
+
const b = opts.b != null ? opts.b : 0.75;
|
|
375
|
+
const N = searchIndex.doc_count;
|
|
376
|
+
|
|
377
|
+
// Average document length (for length normalization).
|
|
378
|
+
let totalLen = 0;
|
|
379
|
+
for (const d of Object.values(searchIndex.documents)) totalLen += d.length || 0;
|
|
380
|
+
const avgdl = N > 0 ? totalLen / N : 0;
|
|
381
|
+
|
|
382
|
+
// BM25 IDF per (deduped) query term, floored at 0 so a term present in nearly
|
|
383
|
+
// every document can't contribute a negative score.
|
|
384
|
+
const qTerms = [...new Set(queryTokens)];
|
|
385
|
+
const idf = {};
|
|
386
|
+
for (const t of qTerms) {
|
|
387
|
+
const df = searchIndex.df[t] || 0;
|
|
388
|
+
idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const raw = {};
|
|
392
|
+
let maxScore = 0;
|
|
393
|
+
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
394
|
+
const dl = doc.length || 0;
|
|
395
|
+
let score = 0;
|
|
396
|
+
for (const t of qTerms) {
|
|
397
|
+
const f = doc.tf[t] || 0;
|
|
398
|
+
if (f === 0) continue;
|
|
399
|
+
const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
|
|
400
|
+
score += idf[t] * (f * (k1 + 1)) / denom;
|
|
401
|
+
}
|
|
402
|
+
if (score > 0) {
|
|
403
|
+
raw[memoryId] = score;
|
|
404
|
+
if (score > maxScore) maxScore = score;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Normalize to 0..1 by the top score — preserves ranking and keeps the [0,1]
|
|
409
|
+
// contract the recall scorer's relevance blend expects.
|
|
410
|
+
const scores = {};
|
|
411
|
+
if (maxScore > 0) {
|
|
412
|
+
for (const [id, s] of Object.entries(raw)) {
|
|
413
|
+
scores[id] = Math.round((s / maxScore) * 1000) / 1000;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
return scores;
|
|
417
|
+
}
|
|
418
|
+
|
|
351
419
|
/**
|
|
352
420
|
* Rebuild the entire search index from memory files on disk.
|
|
353
421
|
*
|
|
@@ -385,6 +453,29 @@ function rebuildIndex(brainDir, index) {
|
|
|
385
453
|
return searchIndex;
|
|
386
454
|
}
|
|
387
455
|
|
|
456
|
+
/**
|
|
457
|
+
* Is the persisted search index out of sync with index.json?
|
|
458
|
+
*
|
|
459
|
+
* The search index drifts whenever memories change outside memorize's
|
|
460
|
+
* addDocument path (forget, sleep/consolidate, sync pull, manual edit) — and a
|
|
461
|
+
* present-but-empty index silently makes every relevance score 0. Both `recall`
|
|
462
|
+
* and `session-start` must rebuild when the doc SET (not mere presence) drifts.
|
|
463
|
+
*
|
|
464
|
+
* @param {Object|null} searchIndex - parsed search-index.json (or null if absent)
|
|
465
|
+
* @param {Object} index - parsed index.json
|
|
466
|
+
* @returns {boolean} true when a rebuild is required
|
|
467
|
+
*/
|
|
468
|
+
function isSearchIndexStale(searchIndex, index) {
|
|
469
|
+
if (!searchIndex || !searchIndex.documents) return true;
|
|
470
|
+
const memIds = Object.keys((index && index.memories) || {});
|
|
471
|
+
const docCount = searchIndex.doc_count != null
|
|
472
|
+
? searchIndex.doc_count
|
|
473
|
+
: Object.keys(searchIndex.documents).length;
|
|
474
|
+
if (docCount !== memIds.length) return true;
|
|
475
|
+
for (const id of memIds) if (!(id in searchIndex.documents)) return true;
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
|
|
388
479
|
module.exports = {
|
|
389
480
|
// Core primitives
|
|
390
481
|
tokenize,
|
|
@@ -401,7 +492,9 @@ module.exports = {
|
|
|
401
492
|
removeDocument,
|
|
402
493
|
computeIdf,
|
|
403
494
|
search,
|
|
495
|
+
bm25Search,
|
|
404
496
|
rebuildIndex,
|
|
497
|
+
isSearchIndexStale,
|
|
405
498
|
// Constants
|
|
406
499
|
SEARCH_INDEX_FILE,
|
|
407
500
|
STOPWORDS,
|