brain-memory 0.1.0-beta.26 → 0.1.0-beta.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scorer.js CHANGED
@@ -365,16 +365,21 @@ function computeSpreadingActivationBatch(scoredMemories, associations, maxDepth
365
365
  * @returns {Object[]} Scored and sorted memories (highest first)
366
366
  */
367
367
  function rankMemories(memories, relevanceFn, options = {}) {
368
+ // Guard every term against NaN (e.g. a missing/invalid last_accessed or
369
+ // strength) — a single NaN score sorts non-deterministically and can outrank
370
+ // valid results.
371
+ const fin = (x) => (Number.isFinite(x) ? x : 0);
372
+
368
373
  // First pass: compute base scores
369
374
  const scored = memories.map((mem) => {
370
- const decayedStrength = computeDecayedStrength(
375
+ const decayedStrength = fin(computeDecayedStrength(
371
376
  mem.strength,
372
377
  mem.decay_rate,
373
378
  mem.last_accessed,
374
379
  !!(mem.stable || mem.pinned) // pinned ⇒ implicitly stable (decay-exempt)
375
- );
376
- const recencyBonus = computeRecencyBonus(mem.last_accessed);
377
- const relevance = relevanceFn(mem);
380
+ ));
381
+ const recencyBonus = fin(computeRecencyBonus(mem.last_accessed));
382
+ const relevance = fin(relevanceFn(mem));
378
383
 
379
384
  return {
380
385
  ...mem,
@@ -420,12 +425,12 @@ function rankMemories(memories, relevanceFn, options = {}) {
420
425
  if (mem.context_match > 0) extras.contextMatch = mem.context_match;
421
426
  if (mem.salience != null) extras.salience = mem.salience;
422
427
 
423
- const score = computeRecallScore(
428
+ const score = fin(computeRecallScore(
424
429
  mem.relevance,
425
430
  mem.decayed_strength,
426
431
  mem.recency_bonus,
427
432
  extras
428
- );
433
+ ));
429
434
 
430
435
  return {
431
436
  ...mem,
@@ -437,7 +442,9 @@ function rankMemories(memories, relevanceFn, options = {}) {
437
442
  score: Math.round(score * 1000) / 1000,
438
443
  };
439
444
  })
440
- .sort((a, b) => b.score - a.score);
445
+ // Deterministic order: score desc, then id asc to break ties (scores are
446
+ // rounded to 3dp, which manufactures ties at scale).
447
+ .sort((a, b) => (b.score - a.score) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
441
448
  }
442
449
 
443
450
  module.exports = {
package/src/tfidf.js CHANGED
@@ -348,6 +348,74 @@ function search(searchIndex, query) {
348
348
  return scores;
349
349
  }
350
350
 
351
+ /**
352
+ * BM25 relevance scoring for a query against all indexed memories.
353
+ *
354
+ * Fixes two TF-IDF-cosine weaknesses that buried long, detailed memories under
355
+ * short distractors: BM25 SATURATES term frequency (k1) and normalizes document
356
+ * length GENTLY (b), instead of dividing TF by full length and cosine by full
357
+ * magnitude (which penalized the richer oracle memories for being longer). The
358
+ * field weighting baked into the index (title 3x, tags 2x via buildMemoryText)
359
+ * is preserved. Scores are normalized to 0..1 by the top score so they blend
360
+ * cleanly into the scorer's recall formula (which expects relevance in [0,1]).
361
+ *
362
+ * @param {Object} searchIndex - The search index
363
+ * @param {string} query - The search query
364
+ * @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
365
+ * @returns {Object} Map of memoryId → normalized relevance score (0.0-1.0)
366
+ */
367
+ function bm25Search(searchIndex, query, opts = {}) {
368
+ if (!searchIndex || searchIndex.doc_count === 0) return {};
369
+
370
+ const queryTokens = tokenize(query);
371
+ if (queryTokens.length === 0) return {};
372
+
373
+ const k1 = opts.k1 != null ? opts.k1 : 1.5;
374
+ const b = opts.b != null ? opts.b : 0.75;
375
+ const N = searchIndex.doc_count;
376
+
377
+ // Average document length (for length normalization).
378
+ let totalLen = 0;
379
+ for (const d of Object.values(searchIndex.documents)) totalLen += d.length || 0;
380
+ const avgdl = N > 0 ? totalLen / N : 0;
381
+
382
+ // BM25 IDF per (deduped) query term, floored at 0 so a term present in nearly
383
+ // every document can't contribute a negative score.
384
+ const qTerms = [...new Set(queryTokens)];
385
+ const idf = {};
386
+ for (const t of qTerms) {
387
+ const df = searchIndex.df[t] || 0;
388
+ idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
389
+ }
390
+
391
+ const raw = {};
392
+ let maxScore = 0;
393
+ for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
394
+ const dl = doc.length || 0;
395
+ let score = 0;
396
+ for (const t of qTerms) {
397
+ const f = doc.tf[t] || 0;
398
+ if (f === 0) continue;
399
+ const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
400
+ score += idf[t] * (f * (k1 + 1)) / denom;
401
+ }
402
+ if (score > 0) {
403
+ raw[memoryId] = score;
404
+ if (score > maxScore) maxScore = score;
405
+ }
406
+ }
407
+
408
+ // Normalize to 0..1 by the top score — preserves ranking and keeps the [0,1]
409
+ // contract the recall scorer's relevance blend expects.
410
+ const scores = {};
411
+ if (maxScore > 0) {
412
+ for (const [id, s] of Object.entries(raw)) {
413
+ scores[id] = Math.round((s / maxScore) * 1000) / 1000;
414
+ }
415
+ }
416
+ return scores;
417
+ }
418
+
351
419
  /**
352
420
  * Rebuild the entire search index from memory files on disk.
353
421
  *
@@ -385,6 +453,29 @@ function rebuildIndex(brainDir, index) {
385
453
  return searchIndex;
386
454
  }
387
455
 
456
+ /**
457
+ * Is the persisted search index out of sync with index.json?
458
+ *
459
+ * The search index drifts whenever memories change outside memorize's
460
+ * addDocument path (forget, sleep/consolidate, sync pull, manual edit) — and a
461
+ * present-but-empty index silently makes every relevance score 0. Both `recall`
462
+ * and `session-start` must rebuild when the doc SET (not mere presence) drifts.
463
+ *
464
+ * @param {Object|null} searchIndex - parsed search-index.json (or null if absent)
465
+ * @param {Object} index - parsed index.json
466
+ * @returns {boolean} true when a rebuild is required
467
+ */
468
+ function isSearchIndexStale(searchIndex, index) {
469
+ if (!searchIndex || !searchIndex.documents) return true;
470
+ const memIds = Object.keys((index && index.memories) || {});
471
+ const docCount = searchIndex.doc_count != null
472
+ ? searchIndex.doc_count
473
+ : Object.keys(searchIndex.documents).length;
474
+ if (docCount !== memIds.length) return true;
475
+ for (const id of memIds) if (!(id in searchIndex.documents)) return true;
476
+ return false;
477
+ }
478
+
388
479
  module.exports = {
389
480
  // Core primitives
390
481
  tokenize,
@@ -401,7 +492,9 @@ module.exports = {
401
492
  removeDocument,
402
493
  computeIdf,
403
494
  search,
495
+ bm25Search,
404
496
  rebuildIndex,
497
+ isSearchIndexStale,
405
498
  // Constants
406
499
  SEARCH_INDEX_FILE,
407
500
  STOPWORDS,