brain-memory 0.1.0-beta.30 → 0.1.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +22 -0
- package/bin/recall.js +8 -0
- package/package.json +1 -1
- package/src/scorer.js +23 -2
- package/src/tfidf.js +22 -3
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.1.0-beta.31] - 2026-07-01
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Recall relevance is now calibrated in absolute terms.** BM25 relevance was
|
|
14
|
+
normalized per-query by the top score, so the best match for *any* query scored
|
|
15
|
+
relevance 1.0 — a nonsense query's one weak term-match produced ten confident-looking
|
|
16
|
+
results (top composite 0.679). Three coordinated changes fix the calibration:
|
|
17
|
+
relevance is scaled by **IDF-weighted query coverage** (a memory matching one term
|
|
18
|
+
of a four-term query can no longer score 1.0); explicit-query recall applies a
|
|
19
|
+
**relevance floor** so zero-relevance memories can't pad the top-N on strength
|
|
20
|
+
alone; and spreading-activation **sources are relevance-gated** in query mode, so a
|
|
21
|
+
strong-but-irrelevant clique can't rescue itself past the floor (associates of
|
|
22
|
+
genuinely relevant memories still surface — that's the feature). Context mode
|
|
23
|
+
(`--context`, session-start) is exempt: topical padding is intended there. On a
|
|
24
|
+
real 177-memory brain the same nonsense query now returns one result with honest
|
|
25
|
+
relevance (0.213), and a 20-query retrieval probe improved from hit@1 90% /
|
|
26
|
+
MRR 0.950 to **hit@1 100% / MRR 1.000** — coverage scaling also demotes
|
|
27
|
+
partial-match distractors below full-coverage targets.
|
|
28
|
+
|
|
9
29
|
## [0.1.0-beta.30] - 2026-07-01
|
|
10
30
|
|
|
11
31
|
### Security
|
package/README.md
CHANGED
|
@@ -406,6 +406,16 @@ score = 0.38 * relevance
|
|
|
406
406
|
- **context_match** (0.14) — How similar the encoding context is to the current session
|
|
407
407
|
- **salience** (0.08) — Emotional/motivational significance
|
|
408
408
|
|
|
409
|
+
Relevance is **calibrated in absolute terms**: BM25 scores are scaled by
|
|
410
|
+
IDF-weighted *query coverage*, so a memory matching one term of a four-term query
|
|
411
|
+
cannot claim relevance 1.0 just for being the best available match. Explicit-query
|
|
412
|
+
recall also applies a **relevance floor** — memories that are neither
|
|
413
|
+
query-relevant nor activated by a relevant associate are excluded rather than
|
|
414
|
+
padding the results on strength alone. A query about things the brain doesn't
|
|
415
|
+
know returns few results with honestly low scores (or none), instead of
|
|
416
|
+
confident-looking noise. Context-mode recall (session start) is exempt, where
|
|
417
|
+
strength-ranked topical padding is the intended behavior.
|
|
418
|
+
|
|
409
419
|
The agent then decides the response strategy:
|
|
410
420
|
- **Single strong match** (top score > 0.7) → Return the full memory
|
|
411
421
|
- **Multiple related** (2-5 candidates > 0.4) → Synthesize a consolidated response
|
|
@@ -689,6 +699,18 @@ Brain Memory ships with a controlled benchmark suite grounded in the 2025-2026 S
|
|
|
689
699
|
|
|
690
700
|
> Results (DeepSeek V4 Pro, 3 runs/arm, June 2026). **Scenario A — retrieval under 1000 distractors:** `brain-full` is the only retriever whose memories let the model succeed — it passes **100%** while both **BM25 (0%)** and a **vector store (0%)** fail to surface the oracle memories — and it does so at the **leanest tokens-per-success (3,199)** of any passing arm. **Scenario D — skills:** loading only the relevant skill passes 100% at **1,580 tok/success, ~31% leaner** than dumping all skill bodies. Scenarios B (continuity) shows brain efficient-and-correct; C (contradiction) and F (abstention) are honest **nulls** — the base model handles them without memory. At n=3 the token gaps are directional, not yet significant; the pass-rate gradient is the result. Full tables: [brainmemory.ai/docs/benchmarks/results](https://brainmemory.ai/docs/benchmarks/results).
|
|
691
701
|
|
|
702
|
+
**Retrieval calibration probe (in-vivo, July 2026).** Separate from the controlled
|
|
703
|
+
suite: a 20-query retrieval probe on a real working brain (177 memories, 1,245
|
|
704
|
+
association edges; tag-derived topical queries with known targets) measured the
|
|
705
|
+
scoring-calibration fix shipped after beta.30. Before: hit@1 90%, MRR 0.950 — and a
|
|
706
|
+
four-term nonsense control query still returned ten results, the top scoring a
|
|
707
|
+
confident-looking 0.679. After query-coverage scaling and the relevance floor:
|
|
708
|
+
**hit@1 100% / hit@3 100% / MRR 1.000**, the nonsense control returns a single result
|
|
709
|
+
with honestly low relevance (0.213), and recall latency stays at ~46 ms (p50).
|
|
710
|
+
*Caveats: n=20, single brain, tag-derived queries favor the index's tag field, not
|
|
711
|
+
judge-graded — this is a calibration probe, not the controlled benchmark above.*
|
|
712
|
+
Method and detail: [brainmemory.ai/docs/benchmarks/results](https://brainmemory.ai/docs/benchmarks/results).
|
|
713
|
+
|
|
692
714
|
Run the benchmarks yourself:
|
|
693
715
|
|
|
694
716
|
```bash
|
package/bin/recall.js
CHANGED
|
@@ -39,6 +39,13 @@ const {
|
|
|
39
39
|
rankMemories,
|
|
40
40
|
} = require('../src/scorer');
|
|
41
41
|
|
|
42
|
+
// Minimum relevance (or spreading bonus) for a memory to appear in explicit-
|
|
43
|
+
// query results. Filters the zero/near-zero-relevance memories that would
|
|
44
|
+
// otherwise pad the top-N purely on strength — an agent trusting those scores
|
|
45
|
+
// injects noise into its context. Context mode (--context) is exempt: there
|
|
46
|
+
// the query is just a topical hint and strength-ranked padding is intended.
|
|
47
|
+
const RELEVANCE_FLOOR = 0.05;
|
|
48
|
+
|
|
42
49
|
function main() {
|
|
43
50
|
const args = parseArgs(process.argv.slice(2));
|
|
44
51
|
|
|
@@ -127,6 +134,7 @@ function main() {
|
|
|
127
134
|
{
|
|
128
135
|
associations: associations || undefined,
|
|
129
136
|
recallContext: Object.keys(recallContext).length > 0 ? recallContext : undefined,
|
|
137
|
+
relevanceFloor: args.context ? undefined : RELEVANCE_FLOOR,
|
|
130
138
|
}
|
|
131
139
|
);
|
|
132
140
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "brain-memory",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.31",
|
|
4
4
|
"description": "A brain-inspired hierarchical memory system for AI coding agents. Deep nested file structure that mimics human memory with time-decay and strength-based recall.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
package/src/scorer.js
CHANGED
|
@@ -362,6 +362,12 @@ function computeSpreadingActivationBatch(scoredMemories, associations, maxDepth
|
|
|
362
362
|
* @param {Object} [options] - Optional scoring parameters
|
|
363
363
|
* @param {Object} [options.associations] - Associations graph for spreading activation
|
|
364
364
|
* @param {Object} [options.recallContext] - Current session context for context matching
|
|
365
|
+
* @param {number} [options.relevanceFloor] - Drop memories whose relevance AND
|
|
366
|
+
* spreading_bonus are both below this floor. Without it, zero-relevance
|
|
367
|
+
* memories pad the results purely on strength/recency/salience — a query
|
|
368
|
+
* about things the brain doesn't know still returns confident-looking hits.
|
|
369
|
+
* Memories rescued by spreading activation survive the floor (associative
|
|
370
|
+
* recall of low-relevance neighbors is a feature, not noise).
|
|
365
371
|
* @returns {Object[]} Scored and sorted memories (highest first)
|
|
366
372
|
*/
|
|
367
373
|
function rankMemories(memories, relevanceFn, options = {}) {
|
|
@@ -393,7 +399,15 @@ function rankMemories(memories, relevanceFn, options = {}) {
|
|
|
393
399
|
|
|
394
400
|
// Second pass: spreading activation (batch — single traversal for all memories)
|
|
395
401
|
if (options.associations) {
|
|
396
|
-
|
|
402
|
+
// With a relevanceFloor set (query mode), only query-relevant memories may
|
|
403
|
+
// act as activation SOURCES — matching the cognitive model (the cue
|
|
404
|
+
// activates matching memories, activation spreads along associations).
|
|
405
|
+
// Without it, a strong-but-irrelevant memory spreads on strength/recency
|
|
406
|
+
// alone and its whole clique rescues itself past the floor below.
|
|
407
|
+
const sources = options.relevanceFloor != null
|
|
408
|
+
? scored.filter((m) => m.relevance >= options.relevanceFloor)
|
|
409
|
+
: scored;
|
|
410
|
+
const scoredSummary = sources.map((m) => ({
|
|
397
411
|
id: m.id,
|
|
398
412
|
score: 0.55 * m.relevance + 0.30 * m.decayed_strength + 0.15 * m.recency_bonus,
|
|
399
413
|
}));
|
|
@@ -417,8 +431,15 @@ function rankMemories(memories, relevanceFn, options = {}) {
|
|
|
417
431
|
}
|
|
418
432
|
}
|
|
419
433
|
|
|
434
|
+
// Relevance floor: gate BEFORE final scoring so strength/salience can't
|
|
435
|
+
// carry an unrelated memory into the results (uses raw, unrounded terms).
|
|
436
|
+
const floor = options.relevanceFloor;
|
|
437
|
+
const gated = floor != null
|
|
438
|
+
? scored.filter((mem) => mem.relevance >= floor || mem.spreading_bonus >= floor)
|
|
439
|
+
: scored;
|
|
440
|
+
|
|
420
441
|
// Final score
|
|
421
|
-
return
|
|
442
|
+
return gated
|
|
422
443
|
.map((mem) => {
|
|
423
444
|
const extras = {};
|
|
424
445
|
if (mem.spreading_bonus > 0) extras.spreadingBonus = mem.spreading_bonus;
|
package/src/tfidf.js
CHANGED
|
@@ -359,6 +359,14 @@ function search(searchIndex, query) {
|
|
|
359
359
|
* is preserved. Scores are normalized to 0..1 by the top score so they blend
|
|
360
360
|
* cleanly into the scorer's recall formula (which expects relevance in [0,1]).
|
|
361
361
|
*
|
|
362
|
+
* Top-score normalization alone is RELATIVE — the best match for a nonsense
|
|
363
|
+
* query would still get relevance 1.0. To keep scores honest in absolute
|
|
364
|
+
* terms, each normalized score is scaled by the doc's IDF-weighted QUERY
|
|
365
|
+
* COVERAGE (share of the query's informative mass the doc actually matched).
|
|
366
|
+
* A doc matching every informative term keeps its score; a doc matching one
|
|
367
|
+
* term of a four-term query is capped near 0.25 no matter how strong that
|
|
368
|
+
* single match is.
|
|
369
|
+
*
|
|
362
370
|
* @param {Object} searchIndex - The search index
|
|
363
371
|
* @param {string} query - The search query
|
|
364
372
|
* @param {Object} [opts] - { k1 = 1.5, b = 0.75 }
|
|
@@ -388,29 +396,40 @@ function bm25Search(searchIndex, query, opts = {}) {
|
|
|
388
396
|
idf[t] = Math.max(0, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
|
|
389
397
|
}
|
|
390
398
|
|
|
399
|
+
// Total informative mass of the query. Terms absent from the corpus (df=0)
|
|
400
|
+
// carry high IDF and can never be matched — a query mostly about things the
|
|
401
|
+
// brain doesn't know SHOULD yield low relevance everywhere.
|
|
402
|
+
let idfTotal = 0;
|
|
403
|
+
for (const t of qTerms) idfTotal += idf[t];
|
|
404
|
+
|
|
391
405
|
const raw = {};
|
|
406
|
+
const coverage = {};
|
|
392
407
|
let maxScore = 0;
|
|
393
408
|
for (const [memoryId, doc] of Object.entries(searchIndex.documents)) {
|
|
394
409
|
const dl = doc.length || 0;
|
|
395
410
|
let score = 0;
|
|
411
|
+
let matchedIdf = 0;
|
|
396
412
|
for (const t of qTerms) {
|
|
397
413
|
const f = doc.tf[t] || 0;
|
|
398
414
|
if (f === 0) continue;
|
|
399
415
|
const denom = f + k1 * (1 - b + b * (avgdl > 0 ? dl / avgdl : 0));
|
|
400
416
|
score += idf[t] * (f * (k1 + 1)) / denom;
|
|
417
|
+
matchedIdf += idf[t];
|
|
401
418
|
}
|
|
402
419
|
if (score > 0) {
|
|
403
420
|
raw[memoryId] = score;
|
|
421
|
+
coverage[memoryId] = idfTotal > 0 ? matchedIdf / idfTotal : 0;
|
|
404
422
|
if (score > maxScore) maxScore = score;
|
|
405
423
|
}
|
|
406
424
|
}
|
|
407
425
|
|
|
408
|
-
// Normalize to 0..1 by the top score
|
|
409
|
-
// contract the recall scorer's relevance blend expects
|
|
426
|
+
// Normalize to 0..1 by the top score (preserves ranking, keeps the [0,1]
|
|
427
|
+
// contract the recall scorer's relevance blend expects), then scale by
|
|
428
|
+
// query coverage so the absolute match quality survives normalization.
|
|
410
429
|
const scores = {};
|
|
411
430
|
if (maxScore > 0) {
|
|
412
431
|
for (const [id, s] of Object.entries(raw)) {
|
|
413
|
-
scores[id] = Math.round((s / maxScore) * 1000) / 1000;
|
|
432
|
+
scores[id] = Math.round((s / maxScore) * coverage[id] * 1000) / 1000;
|
|
414
433
|
}
|
|
415
434
|
}
|
|
416
435
|
return scores;
|