agent-working-memory 0.5.4 → 0.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +87 -46
  2. package/dist/api/routes.d.ts.map +1 -1
  3. package/dist/api/routes.js +21 -5
  4. package/dist/api/routes.js.map +1 -1
  5. package/dist/cli.js +67 -67
  6. package/dist/coordination/index.d.ts +11 -0
  7. package/dist/coordination/index.d.ts.map +1 -0
  8. package/dist/coordination/index.js +39 -0
  9. package/dist/coordination/index.js.map +1 -0
  10. package/dist/coordination/mcp-tools.d.ts +8 -0
  11. package/dist/coordination/mcp-tools.d.ts.map +1 -0
  12. package/dist/coordination/mcp-tools.js +216 -0
  13. package/dist/coordination/mcp-tools.js.map +1 -0
  14. package/dist/coordination/routes.d.ts +9 -0
  15. package/dist/coordination/routes.d.ts.map +1 -0
  16. package/dist/coordination/routes.js +434 -0
  17. package/dist/coordination/routes.js.map +1 -0
  18. package/dist/coordination/schema.d.ts +12 -0
  19. package/dist/coordination/schema.d.ts.map +1 -0
  20. package/dist/coordination/schema.js +91 -0
  21. package/dist/coordination/schema.js.map +1 -0
  22. package/dist/coordination/schemas.d.ts +208 -0
  23. package/dist/coordination/schemas.d.ts.map +1 -0
  24. package/dist/coordination/schemas.js +109 -0
  25. package/dist/coordination/schemas.js.map +1 -0
  26. package/dist/coordination/stale.d.ts +25 -0
  27. package/dist/coordination/stale.d.ts.map +1 -0
  28. package/dist/coordination/stale.js +53 -0
  29. package/dist/coordination/stale.js.map +1 -0
  30. package/dist/index.js +21 -3
  31. package/dist/index.js.map +1 -1
  32. package/dist/mcp.js +90 -79
  33. package/dist/mcp.js.map +1 -1
  34. package/dist/storage/sqlite.d.ts +3 -0
  35. package/dist/storage/sqlite.d.ts.map +1 -1
  36. package/dist/storage/sqlite.js +285 -281
  37. package/dist/storage/sqlite.js.map +1 -1
  38. package/package.json +55 -55
  39. package/src/api/index.ts +3 -3
  40. package/src/api/routes.ts +551 -536
  41. package/src/cli.ts +397 -397
  42. package/src/coordination/index.ts +47 -0
  43. package/src/coordination/mcp-tools.ts +313 -0
  44. package/src/coordination/routes.ts +656 -0
  45. package/src/coordination/schema.ts +94 -0
  46. package/src/coordination/schemas.ts +136 -0
  47. package/src/coordination/stale.ts +89 -0
  48. package/src/core/decay.ts +63 -63
  49. package/src/core/embeddings.ts +88 -88
  50. package/src/core/hebbian.ts +93 -93
  51. package/src/core/index.ts +5 -5
  52. package/src/core/logger.ts +36 -36
  53. package/src/core/query-expander.ts +66 -66
  54. package/src/core/reranker.ts +101 -101
  55. package/src/engine/activation.ts +656 -656
  56. package/src/engine/connections.ts +103 -103
  57. package/src/engine/consolidation-scheduler.ts +125 -125
  58. package/src/engine/eval.ts +102 -102
  59. package/src/engine/eviction.ts +101 -101
  60. package/src/engine/index.ts +8 -8
  61. package/src/engine/retraction.ts +100 -100
  62. package/src/engine/staging.ts +74 -74
  63. package/src/index.ts +137 -121
  64. package/src/mcp.ts +1024 -1013
  65. package/src/storage/index.ts +3 -3
  66. package/src/storage/sqlite.ts +968 -963
  67. package/src/types/agent.ts +67 -67
  68. package/src/types/checkpoint.ts +46 -46
  69. package/src/types/engram.ts +217 -217
  70. package/src/types/eval.ts +100 -100
  71. package/src/types/index.ts +6 -6
@@ -1,656 +1,656 @@
1
- // Copyright 2026 Robert Winter / Complete Ideas
2
- // SPDX-License-Identifier: Apache-2.0
3
- /**
4
- * Activation Pipeline — the core retrieval engine.
5
- *
6
- * 10-phase cognitive retrieval pipeline:
7
- * 0. Query expansion (flan-t5-small synonym generation)
8
- * 1. Vector embedding (MiniLM 384d)
9
- * 2. Parallel retrieval (FTS5/BM25 + vector pool)
10
- * 3. Scoring (BM25, Jaccard, z-score vector, entity-bridge boost)
11
- * 4. Rocchio pseudo-relevance feedback (expand + re-search BM25)
12
- * 5. ACT-R temporal decay
13
- * 6. Hebbian boost (co-activation strength)
14
- * 7. Composite scoring with confidence gating
15
- * 8. Beam search graph walk (depth 2, hop penalty)
16
- * 9. Cross-encoder reranking (ms-marco-MiniLM, adaptive blend)
17
- * 10. Abstention gate
18
- *
19
- * Logs activation events for eval metrics.
20
- */
21
-
22
- import { randomUUID } from 'node:crypto';
23
- import { baseLevelActivation, softplus } from '../core/decay.js';
24
- import { strengthenAssociation, CoActivationBuffer } from '../core/hebbian.js';
25
- import { embed, cosineSimilarity } from '../core/embeddings.js';
26
- import { rerank } from '../core/reranker.js';
27
- import { expandQuery } from '../core/query-expander.js';
28
- import type {
29
- Engram, ActivationResult, ActivationQuery, Association, PhaseScores,
30
- } from '../types/index.js';
31
- import type { EngramStore } from '../storage/sqlite.js';
32
-
33
- /**
34
- * Common English stopwords — filtered from similarity calculations.
35
- * These words carry no semantic signal for memory retrieval.
36
- */
37
- const STOPWORDS = new Set([
38
- 'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
39
- 'her', 'was', 'one', 'our', 'out', 'has', 'have', 'been', 'from', 'that',
40
- 'this', 'with', 'they', 'will', 'each', 'make', 'like', 'then', 'than',
41
- 'them', 'some', 'what', 'when', 'where', 'which', 'who', 'how', 'use',
42
- 'into', 'does', 'also', 'just', 'more', 'over', 'such', 'only', 'very',
43
- 'about', 'after', 'being', 'between', 'could', 'during', 'before',
44
- 'should', 'would', 'their', 'there', 'these', 'those', 'through',
45
- 'because', 'using', 'other',
46
- ]);
47
-
48
- function tokenize(text: string): Set<string> {
49
- return new Set(
50
- text.toLowerCase()
51
- .split(/\s+/)
52
- .filter(w => w.length > 2 && !STOPWORDS.has(w))
53
- );
54
- }
55
-
56
- /**
57
- * Jaccard similarity between two word sets: |intersection| / |union|
58
- */
59
- function jaccard(a: Set<string>, b: Set<string>): number {
60
- if (a.size === 0 || b.size === 0) return 0;
61
- let intersection = 0;
62
- for (const w of a) {
63
- if (b.has(w)) intersection++;
64
- }
65
- const union = a.size + b.size - intersection;
66
- return union > 0 ? intersection / union : 0;
67
- }
68
-
69
- export class ActivationEngine {
70
- private store: EngramStore;
71
- private coActivationBuffer: CoActivationBuffer;
72
-
73
- constructor(store: EngramStore) {
74
- this.store = store;
75
- this.coActivationBuffer = new CoActivationBuffer(50);
76
- }
77
-
78
- /**
79
- * Activate — retrieve the most cognitively relevant engrams for a context.
80
- */
81
- async activate(query: ActivationQuery): Promise<ActivationResult[]> {
82
- const startTime = performance.now();
83
- const limit = query.limit ?? 10;
84
- const minScore = query.minScore ?? 0.01; // Default: filter out zero-relevance results
85
- const useReranker = query.useReranker ?? true;
86
- const useExpansion = query.useExpansion ?? true;
87
- const abstentionThreshold = query.abstentionThreshold ?? 0;
88
-
89
- // Phase -1: Coref expansion — if query has pronouns, append recent entity names
90
- // Helps conversational recall where "she/he/they/it" refers to a named entity.
91
- let queryContext = query.context;
92
- const pronounPattern = /\b(she|he|they|her|his|him|their|it|that|this|there)\b/i;
93
- if (pronounPattern.test(queryContext)) {
94
- // Pull recent entity names from the agent's most-accessed memories
95
- try {
96
- const recentEntities = this.store.getEngramsByAgent(query.agentId, 'active')
97
- .sort((a, b) => b.accessCount - a.accessCount)
98
- .slice(0, 10)
99
- .flatMap(e => e.tags.filter(t => t.length >= 3 && !/^(session-|low-|D\d)/.test(t)))
100
- .filter((v, i, a) => a.indexOf(v) === i)
101
- .slice(0, 5);
102
- if (recentEntities.length > 0) {
103
- queryContext = `${queryContext} ${recentEntities.join(' ')}`;
104
- }
105
- } catch { /* non-fatal */ }
106
- }
107
-
108
- // Phase 0: Query expansion — add related terms to improve BM25 recall
109
- let searchContext = queryContext;
110
- if (useExpansion) {
111
- try {
112
- searchContext = await expandQuery(query.context);
113
- } catch {
114
- // Expansion unavailable — use original query
115
- }
116
- }
117
-
118
- // Phase 1: Embed query for vector similarity (uses coref-expanded context)
119
- let queryEmbedding: number[] | null = null;
120
- try {
121
- queryEmbedding = await embed(queryContext);
122
- } catch {
123
- // Embedding unavailable — fall back to text-only matching
124
- }
125
-
126
- // Phase 2: Parallel retrieval — dual BM25 + all active engrams
127
- // Two-pass BM25: (1) keyword-stripped query for precision, (2) expanded query for recall.
128
- const keywordQuery = Array.from(tokenize(query.context)).join(' ');
129
- const bm25Keyword = keywordQuery.length > 2
130
- ? this.store.searchBM25WithRank(query.agentId, keywordQuery, limit * 3)
131
- : [];
132
- const bm25Expanded = this.store.searchBM25WithRank(query.agentId, searchContext, limit * 3);
133
-
134
- // Merge: take the best BM25 score per engram from either pass
135
- const bm25ScoreMap = new Map<string, number>();
136
- const bm25EngramMap = new Map<string, any>();
137
- for (const r of [...bm25Keyword, ...bm25Expanded]) {
138
- const existing = bm25ScoreMap.get(r.engram.id) ?? 0;
139
- if (r.bm25Score > existing) {
140
- bm25ScoreMap.set(r.engram.id, r.bm25Score);
141
- bm25EngramMap.set(r.engram.id, r.engram);
142
- }
143
- }
144
- const bm25Ranked = Array.from(bm25EngramMap.entries()).map(([id, engram]) => ({
145
- engram, bm25Score: bm25ScoreMap.get(id) ?? 0,
146
- }));
147
-
148
- const allActive = this.store.getEngramsByAgent(
149
- query.agentId,
150
- query.includeStaging ? undefined : 'active',
151
- query.includeRetracted ?? false
152
- );
153
-
154
- // Merge candidates (deduplicate)
155
- const candidateMap = new Map<string, Engram>();
156
- for (const r of bm25Ranked) candidateMap.set(r.engram.id, r.engram);
157
- for (const e of allActive) candidateMap.set(e.id, e);
158
- const candidates = Array.from(candidateMap.values());
159
-
160
- if (candidates.length === 0) return [];
161
-
162
- // Tokenize query once
163
- const queryTokens = tokenize(query.context);
164
-
165
- // Phase 3a: Compute raw cosine similarities for adaptive normalization
166
- const rawCosineSims = new Map<string, number>();
167
- if (queryEmbedding) {
168
- for (const engram of candidates) {
169
- if (engram.embedding) {
170
- rawCosineSims.set(engram.id, cosineSimilarity(queryEmbedding, engram.embedding));
171
- }
172
- }
173
- }
174
-
175
- // Compute distribution stats for model-agnostic normalization
176
- const simValues = Array.from(rawCosineSims.values());
177
- const simMean = simValues.length > 0
178
- ? simValues.reduce((a, b) => a + b, 0) / simValues.length : 0;
179
- const rawStdDev = simValues.length > 1
180
- ? Math.sqrt(simValues.reduce((sum, s) => sum + (s - simMean) ** 2, 0) / simValues.length) : 0.15;
181
- // Floor stddev at 0.10 to prevent z-score inflation with small candidate pools
182
- const simStdDev = Math.max(rawStdDev, 0.10);
183
-
184
- // Phase 3b: Score each candidate with per-phase breakdown
185
- const scored = candidates.map(engram => {
186
- const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
187
- const associations = this.store.getAssociationsFor(engram.id);
188
-
189
- // --- Text relevance (keyword signals) ---
190
-
191
- // Signal 1: BM25 continuous score (0-1, from FTS5 rank)
192
- const bm25Score = bm25ScoreMap.get(engram.id) ?? 0;
193
-
194
- // Signal 2: Jaccard similarity with stopword filtering
195
- const conceptTokens = tokenize(engram.concept);
196
- const contentTokens = tokenize(engram.content);
197
- const conceptJaccard = jaccard(queryTokens, conceptTokens);
198
- const contentJaccard = jaccard(queryTokens, contentTokens);
199
- const jaccardScore = 0.6 * conceptJaccard + 0.4 * contentJaccard;
200
-
201
- // Signal 3: Concept exact match bonus (up to 0.3)
202
- const conceptOverlap = conceptTokens.size > 0
203
- ? [...conceptTokens].filter(w => queryTokens.has(w)).length / conceptTokens.size
204
- : 0;
205
- const conceptBonus = conceptOverlap * 0.3;
206
-
207
- const keywordMatch = Math.min(Math.max(bm25Score, jaccardScore) + conceptBonus, 1.0);
208
-
209
- // --- Vector similarity (semantic signal) ---
210
- // Two-stage: absolute floor prevents noise, then z-score ranks within matches.
211
- // Stage 1: Raw cosine must exceed mean + 1 stddev (absolute relevance gate)
212
- // Stage 2: Z-score maps relative position to 0-1 for ranking quality
213
- let vectorMatch = 0;
214
- const rawSim = rawCosineSims.get(engram.id);
215
- if (rawSim !== undefined) {
216
- const zScore = (rawSim - simMean) / simStdDev;
217
- // Gate: must be at least 1 stddev above mean to be considered a match
218
- if (zScore > 1.0) {
219
- // Map z=1..3 → 0..1 linearly
220
- vectorMatch = Math.min(1, (zScore - 1.0) / 2.0);
221
- }
222
- }
223
-
224
- // Combined text match: best of keyword and vector signals
225
- const textMatch = Math.max(keywordMatch, vectorMatch);
226
-
227
- // --- Temporal signals ---
228
-
229
- // ACT-R decay — confidence + replay modulated (synaptic tagging)
230
- // High-confidence memories decay slower. Heavily-accessed memories also resist decay.
231
- // Default exponent: 0.5. High confidence (0.8+): 0.3. High access (10+): further -0.05.
232
- const confMod = 0.2 * Math.max(0, (engram.confidence - 0.5) / 0.5);
233
- const replayMod = Math.min(0.1, 0.05 * Math.log1p(engram.accessCount));
234
- const decayExponent = Math.max(0.2, 0.5 - confMod - replayMod);
235
- const decayScore = baseLevelActivation(engram.accessCount, ageDays, decayExponent);
236
-
237
- // Hebbian boost from associations — capped to prevent popular memories
238
- // from dominating regardless of query relevance
239
- const rawHebbian = associations.length > 0
240
- ? associations.reduce((sum, a) => sum + a.weight, 0) / associations.length
241
- : 0;
242
- const hebbianBoost = Math.min(rawHebbian, 0.5);
243
-
244
- // Centrality signal — well-connected memories (high weighted degree)
245
- // get a small boost. This makes consolidation edges matter for retrieval.
246
- // Log-scaled to prevent hub domination: 10 edges ≈ 0.05 boost, 50 ≈ 0.08
247
- const weightedDegree = associations.reduce((sum, a) => sum + a.weight, 0);
248
- const centralityBoost = associations.length > 0
249
- ? Math.min(0.1, 0.03 * Math.log1p(weightedDegree))
250
- : 0;
251
-
252
- // Confidence gate — multiplicative quality signal
253
- const confidenceGate = engram.confidence;
254
-
255
- // Feedback bonus — memories confirmed useful via explicit feedback get a
256
- // direct additive boost. Models how a senior dev "just knows" certain things
257
- // are important. Confidence > 0.6 means at least 2+ positive feedbacks.
258
- // Scales: conf 0.6→0.03, 0.7→0.06, 0.8→0.09, 1.0→0.15
259
- const feedbackBonus = engram.confidence > 0.55
260
- ? Math.min(0.15, 0.3 * Math.max(0, engram.confidence - 0.5))
261
- : 0;
262
-
263
- // --- Composite score: relevance-gated additive ---
264
- // Text relevance must be present for temporal signals to contribute.
265
- // Without text relevance, a memory shouldn't activate regardless of recency.
266
- // Temporal contribution scales with text relevance (weak match = weak temporal boost).
267
- const temporalNorm = Math.min(softplus(decayScore + hebbianBoost), 3.0) / 3.0;
268
- const relevanceGate = textMatch > 0.1 ? textMatch : 0.0; // Proportional gate
269
- const composite = (0.6 * textMatch + 0.4 * temporalNorm * relevanceGate + centralityBoost * relevanceGate + feedbackBonus * relevanceGate) * confidenceGate;
270
-
271
- const phaseScores: PhaseScores = {
272
- textMatch,
273
- vectorMatch,
274
- decayScore,
275
- hebbianBoost,
276
- graphBoost: 0, // Filled in phase 5
277
- confidenceGate,
278
- composite,
279
- rerankerScore: 0, // Filled in phase 7
280
- };
281
-
282
- return { engram, score: composite, phaseScores, associations };
283
- });
284
-
285
- // Phase 3.5: Rocchio pseudo-relevance feedback — expand query with top result terms
286
- // then re-search BM25 to find candidates that keyword search missed
287
- const preSorted = scored.sort((a, b) => b.score - a.score);
288
- const topForFeedback = preSorted.slice(0, 3).filter(r => r.phaseScores.textMatch > 0.1);
289
- if (topForFeedback.length > 0) {
290
- const feedbackTerms = new Set<string>();
291
- for (const item of topForFeedback) {
292
- const tokens = tokenize(item.engram.content);
293
- for (const t of tokens) {
294
- if (!queryTokens.has(t) && t.length >= 4) feedbackTerms.add(t);
295
- }
296
- }
297
- // Take top 5 feedback terms and re-search
298
- const extraTerms = Array.from(feedbackTerms).slice(0, 5).join(' ');
299
- if (extraTerms) {
300
- const feedbackBM25 = this.store.searchBM25WithRank(query.agentId, `${searchContext} ${extraTerms}`, limit * 2);
301
- for (const r of feedbackBM25) {
302
- if (!candidateMap.has(r.engram.id)) {
303
- candidateMap.set(r.engram.id, r.engram);
304
- // Score the new candidate
305
- const engram = r.engram;
306
- const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
307
- const associations = this.store.getAssociationsFor(engram.id);
308
- const cTokens = tokenize(engram.concept);
309
- const ctTokens = tokenize(engram.content);
310
- const cJac = jaccard(queryTokens, cTokens);
311
- const ctJac = jaccard(queryTokens, ctTokens);
312
- const jSc = 0.6 * cJac + 0.4 * ctJac;
313
- const cOvlp = cTokens.size > 0 ? [...cTokens].filter(w => queryTokens.has(w)).length / cTokens.size : 0;
314
- const km = Math.min(Math.max(r.bm25Score, jSc) + cOvlp * 0.3, 1.0);
315
- let vm = 0;
316
- const rs = rawCosineSims.get(engram.id) ?? (queryEmbedding && engram.embedding ? cosineSimilarity(queryEmbedding, engram.embedding) : 0);
317
- if (rs) {
318
- const z = (rs - simMean) / simStdDev;
319
- if (z > 1.0) vm = Math.min(1, (z - 1.0) / 2.0);
320
- }
321
- const tm = Math.max(km, vm);
322
- const ds = baseLevelActivation(engram.accessCount, ageDays);
323
- const rh = associations.length > 0 ? Math.min(associations.reduce((s, a) => s + a.weight, 0) / associations.length, 0.5) : 0;
324
- const tn = Math.min(softplus(ds + rh), 3.0) / 3.0;
325
- const rg = tm > 0.1 ? tm : 0.0;
326
- const comp = (0.6 * tm + 0.4 * tn * rg) * engram.confidence;
327
- scored.push({ engram, score: comp, phaseScores: { textMatch: tm, vectorMatch: vm, decayScore: ds, hebbianBoost: rh, graphBoost: 0, confidenceGate: engram.confidence, composite: comp, rerankerScore: 0 }, associations });
328
- }
329
- }
330
- }
331
- }
332
-
333
- // Phase 3.7: Entity-Bridge boost — boost scored candidates that share entity tags
334
- // with the most query-relevant result. Only bridge from the single best text-match
335
- // to avoid pulling in unrelated entities from tangentially-matching results.
336
- {
337
- // Find the result with the highest textMatch (most query-relevant, not just highest score)
338
- // Gate: only bridge when anchor has meaningful text relevance (> 0.15)
339
- // Adaptive: scale bridge boost inversely with candidate pool size to prevent
340
- // over-boosting in large memory pools where many items share entity tags
341
- const sortedByTextMatch = scored
342
- .filter(r => r.phaseScores.textMatch > 0.15)
343
- .sort((a, b) => b.phaseScores.textMatch - a.phaseScores.textMatch);
344
-
345
- // Bridge from top 2 text-matched results (IDF handles weighting)
346
- const bridgeAnchors = sortedByTextMatch.slice(0, 2);
347
-
348
- if (bridgeAnchors.length > 0) {
349
- const entityTags = new Set<string>();
350
- const anchorIds = new Set(bridgeAnchors.map(r => r.engram.id));
351
-
352
- for (const item of bridgeAnchors) {
353
- for (const tag of item.engram.tags) {
354
- const t = tag.toLowerCase();
355
- // Skip non-entity tags: turn IDs, session tags, dialogue IDs, generic speaker labels
356
- if (/^t\d+$/.test(t) || t.startsWith('session-') || t.startsWith('dia_') || t.length < 3) continue;
357
- if (/^speaker\d*$/.test(t)) continue; // Generic speaker labels are too broad
358
- entityTags.add(t);
359
- }
360
- }
361
-
362
- // Document frequency filter: remove tags appearing in >30% of items (too common)
363
- // This prevents speaker names in 2-person conversations from being used as bridges
364
- if (entityTags.size > 0 && scored.length > 10) {
365
- const tagFreqs = new Map<string, number>();
366
- for (const item of scored) {
367
- const seen = new Set<string>();
368
- for (const tag of item.engram.tags) {
369
- const t = tag.toLowerCase();
370
- if (entityTags.has(t) && !seen.has(t)) {
371
- seen.add(t);
372
- tagFreqs.set(t, (tagFreqs.get(t) ?? 0) + 1);
373
- }
374
- }
375
- }
376
- const maxFreq = scored.length * 0.30;
377
- for (const [tag, freq] of tagFreqs) {
378
- if (freq > maxFreq) entityTags.delete(tag);
379
- }
380
- }
381
-
382
- if (entityTags.size > 0) {
383
- for (const item of scored) {
384
- if (anchorIds.has(item.engram.id)) continue;
385
-
386
- const engramTags = new Set(item.engram.tags.map((t: string) => t.toLowerCase()));
387
- let sharedEntities = 0;
388
- for (const et of entityTags) {
389
- if (engramTags.has(et)) sharedEntities++;
390
- }
391
-
392
- if (sharedEntities > 0) {
393
- // Flat bridge boost per shared entity
394
- const bridgeBoost = Math.min(sharedEntities * 0.15, 0.4);
395
- item.score += bridgeBoost;
396
- item.phaseScores.composite += bridgeBoost;
397
- item.phaseScores.graphBoost += bridgeBoost;
398
- }
399
- }
400
- }
401
- }
402
- }
403
-
404
- // Phase 4+5: Graph walk — boost engrams connected to high-scoring ones
405
- // Only walk from engrams that had text relevance (composite > 0 pre-walk)
406
- const sorted = scored.sort((a, b) => b.score - a.score);
407
- const topN = sorted.slice(0, limit * 3);
408
- this.graphWalk(topN, 2, 0.3);
409
-
410
- // Phase 6: Initial filter and sort for re-ranking pool
411
- const pool = topN
412
- .filter(r => r.score >= minScore)
413
- .sort((a, b) => b.score - a.score);
414
-
415
- // Phase 7: Cross-encoder re-ranking — scores (query, passage) pairs directly
416
- // Widens the pool to find relevant results that keyword matching missed
417
- const rerankPool = pool.slice(0, Math.max(limit * 3, 30));
418
-
419
- if (useReranker && rerankPool.length > 0) {
420
- try {
421
- const passages = rerankPool.map(r =>
422
- `${r.engram.concept}: ${r.engram.content}`
423
- );
424
- const rerankResults = await rerank(query.context, passages);
425
-
426
- // Adaptive reranker blend (Codex recommendation):
427
- // When BM25/text signals are strong, trust them more; when weak, lean on reranker.
428
- const bm25Max = Math.max(...rerankPool.map(r => r.phaseScores.textMatch));
429
- const rerankWeight = Math.min(0.7, Math.max(0.3, 0.3 + 0.4 * (1 - bm25Max)));
430
- const compositeWeight = 1 - rerankWeight;
431
-
432
- for (const rr of rerankResults) {
433
- const item = rerankPool[rr.index];
434
- item.phaseScores.rerankerScore = rr.score;
435
- item.score = compositeWeight * item.phaseScores.composite + rerankWeight * rr.score;
436
- }
437
- } catch {
438
- // Re-ranker unavailable — keep original scores
439
- }
440
- }
441
-
442
- // Phase 8: Multi-channel OOD detection + agreement gate
443
- // Requires at least 2 of 3 retrieval channels to agree the query is in-domain.
444
- if (rerankPool.length >= 3) {
445
- const topBM25 = Math.max(...rerankPool.map(r => bm25ScoreMap.get(r.engram.id) ?? 0));
446
- const topVector = queryEmbedding
447
- ? Math.max(...rerankPool.map(r => r.phaseScores.vectorMatch))
448
- : 0;
449
- const topReranker = Math.max(...rerankPool.map(r => r.phaseScores.rerankerScore));
450
-
451
- const bm25Ok = topBM25 > 0.3;
452
- const vectorOk = topVector > 0.05;
453
- const rerankerOk = topReranker > 0.25;
454
- const channelsAgreeing = (bm25Ok ? 1 : 0) + (vectorOk ? 1 : 0) + (rerankerOk ? 1 : 0);
455
-
456
- const rerankerScores = rerankPool
457
- .map(r => r.phaseScores.rerankerScore)
458
- .sort((a, b) => b - a);
459
- const margin = rerankerScores.length >= 2
460
- ? rerankerScores[0] - rerankerScores[1]
461
- : rerankerScores[0];
462
-
463
- const maxRawCosine = queryEmbedding && simValues.length > 0
464
- ? Math.max(...simValues)
465
- : 1.0;
466
-
467
- // Stricter gate when caller explicitly requests abstention (e.g., noise filter queries)
468
- const requiredChannels = abstentionThreshold > 0 ? 3 : 2;
469
-
470
- // Hard abstention: fewer than required channels agree AND semantic drift is high
471
- if (channelsAgreeing < requiredChannels && maxRawCosine < (simMean + simStdDev * 1.5)) {
472
- return [];
473
- }
474
-
475
- // Soft penalty: only 1 channel agrees or margin is thin
476
- if (channelsAgreeing < 2 || margin < 0.05) {
477
- // If caller explicitly requested abstention, honor it when agreement is weak
478
- if (abstentionThreshold > 0) {
479
- return [];
480
- }
481
- for (const item of rerankPool) {
482
- item.score *= 0.4;
483
- }
484
- }
485
- }
486
-
487
- // Legacy abstention gate (when explicitly requested)
488
- if (abstentionThreshold > 0 && rerankPool.length >= 3) {
489
- const topRerankerScores = rerankPool
490
- .map(r => r.phaseScores.rerankerScore)
491
- .sort((a, b) => b - a)
492
- .slice(0, 5);
493
- const maxScore = topRerankerScores[0];
494
- const meanScore = topRerankerScores.reduce((s, v) => s + v, 0) / topRerankerScores.length;
495
- const variance = topRerankerScores.reduce((s, v) => s + (v - meanScore) ** 2, 0) / topRerankerScores.length;
496
-
497
- if (maxScore < abstentionThreshold || (maxScore < 0.5 && variance < 0.01)) {
498
- return [];
499
- }
500
- }
501
-
502
- // Phase 8c: Supersession penalty — superseded memories are deprioritized.
503
- // They aren't wrong (that's retraction), just outdated.
504
- for (const item of rerankPool) {
505
- if (item.engram.supersededBy) {
506
- item.score *= 0.15; // Severe down-rank — successor should dominate
507
- }
508
- }
509
-
510
- // Phase 9: Final sort, limit, explain
511
- const results: ActivationResult[] = rerankPool
512
- .sort((a, b) => b.score - a.score)
513
- .slice(0, limit)
514
- .map(r => ({
515
- engram: r.engram,
516
- score: r.score,
517
- phaseScores: r.phaseScores,
518
- why: this.explain(r.phaseScores, r.engram, r.associations),
519
- associations: r.associations,
520
- }));
521
-
522
- const activatedIds = results.map(r => r.engram.id);
523
-
524
- // Side effects: touch, co-activate, Hebbian update (skip for internal/system calls)
525
- if (!query.internal) {
526
- for (const id of activatedIds) {
527
- this.store.touchEngram(id);
528
- }
529
- this.coActivationBuffer.pushBatch(activatedIds);
530
- this.updateHebbianWeights();
531
-
532
- // Log activation event for eval
533
- const latencyMs = performance.now() - startTime;
534
- this.store.logActivationEvent({
535
- id: randomUUID(),
536
- agentId: query.agentId,
537
- timestamp: new Date(),
538
- context: query.context,
539
- resultsReturned: results.length,
540
- topScore: results.length > 0 ? results[0].score : 0,
541
- latencyMs,
542
- engramIds: activatedIds,
543
- });
544
- }
545
-
546
- return results;
547
- }
548
-
549
- /**
550
- * Beam search graph walk — replaces naive BFS.
551
- * Scores paths (not just nodes), uses query-dependent edge filtering,
552
- * and supports deeper exploration with focused beams.
553
- */
554
- private graphWalk(
555
- scored: { engram: Engram; score: number; phaseScores: PhaseScores; associations: Association[] }[],
556
- maxDepth: number,
557
- hopPenalty: number
558
- ): void {
559
- const scoreMap = new Map(scored.map(s => [s.engram.id, s]));
560
- const MAX_TOTAL_BOOST = 0.25; // Slightly higher cap for beam search (deeper paths earn it)
561
- const BEAM_WIDTH = 15;
562
-
563
- // Seed the beam with high-scoring, text-relevant items
564
- const beam = scored
565
- .filter(item => item.phaseScores.textMatch >= 0.15)
566
- .sort((a, b) => b.score - a.score)
567
- .slice(0, BEAM_WIDTH);
568
-
569
- // Track which engrams have been explored (avoid cycles)
570
- const explored = new Set<string>();
571
-
572
- for (let depth = 0; depth < maxDepth; depth++) {
573
- const nextBeam: typeof beam = [];
574
-
575
- for (const item of beam) {
576
- if (explored.has(item.engram.id)) continue;
577
- explored.add(item.engram.id);
578
-
579
- // Get associations — for depth > 0, fetch from store if not in scored set
580
- const associations = item.associations.length > 0
581
- ? item.associations
582
- : this.store.getAssociationsFor(item.engram.id);
583
-
584
- for (const assoc of associations) {
585
- const neighborId = assoc.fromEngramId === item.engram.id
586
- ? assoc.toEngramId
587
- : assoc.fromEngramId;
588
-
589
- if (explored.has(neighborId)) continue;
590
-
591
- const neighbor = scoreMap.get(neighborId);
592
- if (!neighbor) continue;
593
-
594
- // Query-dependent edge filtering: neighbor must have SOME relevance
595
- // (textMatch > 0.05 for deeper hops, relaxed from 0.1)
596
- const relevanceFloor = depth === 0 ? 0.1 : 0.05;
597
- if (neighbor.phaseScores.textMatch < relevanceFloor) continue;
598
-
599
- // Skip if neighbor already at boost cap
600
- if (neighbor.phaseScores.graphBoost >= MAX_TOTAL_BOOST) continue;
601
-
602
- // Path score: source score * edge weight * hop penalty^(depth+1)
603
- const normalizedWeight = Math.min(assoc.weight, 5.0) / 5.0;
604
- const pathScore = item.score * normalizedWeight * Math.pow(hopPenalty, depth + 1);
605
-
606
- const boost = Math.min(pathScore, 0.15, MAX_TOTAL_BOOST - neighbor.phaseScores.graphBoost);
607
- if (boost > 0.001) {
608
- neighbor.score += boost;
609
- neighbor.phaseScores.graphBoost += boost;
610
- nextBeam.push(neighbor);
611
- }
612
- }
613
- }
614
-
615
- // Prune beam for next depth level
616
- if (nextBeam.length === 0) break;
617
- beam.length = 0;
618
- beam.push(...nextBeam
619
- .sort((a, b) => b.score - a.score)
620
- .slice(0, BEAM_WIDTH)
621
- );
622
- }
623
- }
624
-
625
- private updateHebbianWeights(): void {
626
- const pairs = this.coActivationBuffer.getCoActivatedPairs(10_000);
627
- // Deduplicate pairs to prevent repeated strengthening
628
- const seen = new Set<string>();
629
- for (const [a, b] of pairs) {
630
- const key = a < b ? `${a}:${b}` : `${b}:${a}`;
631
- if (seen.has(key)) continue;
632
- seen.add(key);
633
-
634
- const existing = this.store.getAssociation(a, b) ?? this.store.getAssociation(b, a);
635
- const currentWeight = existing?.weight ?? 0.1;
636
- const newWeight = strengthenAssociation(currentWeight);
637
- this.store.upsertAssociation(a, b, newWeight, 'hebbian');
638
- this.store.upsertAssociation(b, a, newWeight, 'hebbian');
639
- }
640
- }
641
-
642
- private explain(phases: PhaseScores, engram: Engram, associations: Association[]): string {
643
- const parts: string[] = [];
644
- parts.push(`composite=${phases.composite.toFixed(3)}`);
645
- if (phases.textMatch > 0) parts.push(`text=${phases.textMatch.toFixed(2)}`);
646
- if (phases.vectorMatch > 0) parts.push(`vector=${phases.vectorMatch.toFixed(2)}`);
647
- parts.push(`decay=${phases.decayScore.toFixed(2)}`);
648
- if (phases.hebbianBoost > 0) parts.push(`hebbian=${phases.hebbianBoost.toFixed(2)}`);
649
- if (phases.graphBoost > 0) parts.push(`graph=${phases.graphBoost.toFixed(2)}`);
650
- if (phases.rerankerScore > 0) parts.push(`reranker=${phases.rerankerScore.toFixed(2)}`);
651
- parts.push(`conf=${phases.confidenceGate.toFixed(2)}`);
652
- parts.push(`access=${engram.accessCount}`);
653
- if (associations.length > 0) parts.push(`edges=${associations.length}`);
654
- return parts.join(' | ');
655
- }
656
- }
1
+ // Copyright 2026 Robert Winter / Complete Ideas
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ /**
4
+ * Activation Pipeline — the core retrieval engine.
5
+ *
6
+ * 10-phase cognitive retrieval pipeline:
7
+ * 0. Query expansion (flan-t5-small synonym generation)
8
+ * 1. Vector embedding (MiniLM 384d)
9
+ * 2. Parallel retrieval (FTS5/BM25 + vector pool)
10
+ * 3. Scoring (BM25, Jaccard, z-score vector, entity-bridge boost)
11
+ * 4. Rocchio pseudo-relevance feedback (expand + re-search BM25)
12
+ * 5. ACT-R temporal decay
13
+ * 6. Hebbian boost (co-activation strength)
14
+ * 7. Composite scoring with confidence gating
15
+ * 8. Beam search graph walk (depth 2, hop penalty)
16
+ * 9. Cross-encoder reranking (ms-marco-MiniLM, adaptive blend)
17
+ * 10. Abstention gate
18
+ *
19
+ * Logs activation events for eval metrics.
20
+ */
21
+
22
+ import { randomUUID } from 'node:crypto';
23
+ import { baseLevelActivation, softplus } from '../core/decay.js';
24
+ import { strengthenAssociation, CoActivationBuffer } from '../core/hebbian.js';
25
+ import { embed, cosineSimilarity } from '../core/embeddings.js';
26
+ import { rerank } from '../core/reranker.js';
27
+ import { expandQuery } from '../core/query-expander.js';
28
+ import type {
29
+ Engram, ActivationResult, ActivationQuery, Association, PhaseScores,
30
+ } from '../types/index.js';
31
+ import type { EngramStore } from '../storage/sqlite.js';
32
+
33
+ /**
34
+ * Common English stopwords — filtered from similarity calculations.
35
+ * These words carry no semantic signal for memory retrieval.
36
+ */
37
+ const STOPWORDS = new Set([
38
+ 'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
39
+ 'her', 'was', 'one', 'our', 'out', 'has', 'have', 'been', 'from', 'that',
40
+ 'this', 'with', 'they', 'will', 'each', 'make', 'like', 'then', 'than',
41
+ 'them', 'some', 'what', 'when', 'where', 'which', 'who', 'how', 'use',
42
+ 'into', 'does', 'also', 'just', 'more', 'over', 'such', 'only', 'very',
43
+ 'about', 'after', 'being', 'between', 'could', 'during', 'before',
44
+ 'should', 'would', 'their', 'there', 'these', 'those', 'through',
45
+ 'because', 'using', 'other',
46
+ ]);
47
+
48
+ function tokenize(text: string): Set<string> {
49
+ return new Set(
50
+ text.toLowerCase()
51
+ .split(/\s+/)
52
+ .filter(w => w.length > 2 && !STOPWORDS.has(w))
53
+ );
54
+ }
55
+
56
+ /**
57
+ * Jaccard similarity between two word sets: |intersection| / |union|
58
+ */
59
+ function jaccard(a: Set<string>, b: Set<string>): number {
60
+ if (a.size === 0 || b.size === 0) return 0;
61
+ let intersection = 0;
62
+ for (const w of a) {
63
+ if (b.has(w)) intersection++;
64
+ }
65
+ const union = a.size + b.size - intersection;
66
+ return union > 0 ? intersection / union : 0;
67
+ }
68
+
69
+ export class ActivationEngine {
70
+ private store: EngramStore;
71
+ private coActivationBuffer: CoActivationBuffer;
72
+
73
+ constructor(store: EngramStore) {
74
+ this.store = store;
75
+ this.coActivationBuffer = new CoActivationBuffer(50);
76
+ }
77
+
78
+ /**
79
+ * Activate — retrieve the most cognitively relevant engrams for a context.
80
+ */
81
+ async activate(query: ActivationQuery): Promise<ActivationResult[]> {
82
+ const startTime = performance.now();
83
+ const limit = query.limit ?? 10;
84
+ const minScore = query.minScore ?? 0.01; // Default: filter out zero-relevance results
85
+ const useReranker = query.useReranker ?? true;
86
+ const useExpansion = query.useExpansion ?? true;
87
+ const abstentionThreshold = query.abstentionThreshold ?? 0;
88
+
89
+ // Phase -1: Coref expansion — if query has pronouns, append recent entity names
90
+ // Helps conversational recall where "she/he/they/it" refers to a named entity.
91
+ let queryContext = query.context;
92
+ const pronounPattern = /\b(she|he|they|her|his|him|their|it|that|this|there)\b/i;
93
+ if (pronounPattern.test(queryContext)) {
94
+ // Pull recent entity names from the agent's most-accessed memories
95
+ try {
96
+ const recentEntities = this.store.getEngramsByAgent(query.agentId, 'active')
97
+ .sort((a, b) => b.accessCount - a.accessCount)
98
+ .slice(0, 10)
99
+ .flatMap(e => e.tags.filter(t => t.length >= 3 && !/^(session-|low-|D\d)/.test(t)))
100
+ .filter((v, i, a) => a.indexOf(v) === i)
101
+ .slice(0, 5);
102
+ if (recentEntities.length > 0) {
103
+ queryContext = `${queryContext} ${recentEntities.join(' ')}`;
104
+ }
105
+ } catch { /* non-fatal */ }
106
+ }
107
+
108
+ // Phase 0: Query expansion — add related terms to improve BM25 recall
109
+ let searchContext = queryContext;
110
+ if (useExpansion) {
111
+ try {
112
+ searchContext = await expandQuery(query.context);
113
+ } catch {
114
+ // Expansion unavailable — use original query
115
+ }
116
+ }
117
+
118
+ // Phase 1: Embed query for vector similarity (uses coref-expanded context)
119
+ let queryEmbedding: number[] | null = null;
120
+ try {
121
+ queryEmbedding = await embed(queryContext);
122
+ } catch {
123
+ // Embedding unavailable — fall back to text-only matching
124
+ }
125
+
126
+ // Phase 2: Parallel retrieval — dual BM25 + all active engrams
127
+ // Two-pass BM25: (1) keyword-stripped query for precision, (2) expanded query for recall.
128
+ const keywordQuery = Array.from(tokenize(query.context)).join(' ');
129
+ const bm25Keyword = keywordQuery.length > 2
130
+ ? this.store.searchBM25WithRank(query.agentId, keywordQuery, limit * 3)
131
+ : [];
132
+ const bm25Expanded = this.store.searchBM25WithRank(query.agentId, searchContext, limit * 3);
133
+
134
+ // Merge: take the best BM25 score per engram from either pass
135
+ const bm25ScoreMap = new Map<string, number>();
136
+ const bm25EngramMap = new Map<string, any>();
137
+ for (const r of [...bm25Keyword, ...bm25Expanded]) {
138
+ const existing = bm25ScoreMap.get(r.engram.id) ?? 0;
139
+ if (r.bm25Score > existing) {
140
+ bm25ScoreMap.set(r.engram.id, r.bm25Score);
141
+ bm25EngramMap.set(r.engram.id, r.engram);
142
+ }
143
+ }
144
+ const bm25Ranked = Array.from(bm25EngramMap.entries()).map(([id, engram]) => ({
145
+ engram, bm25Score: bm25ScoreMap.get(id) ?? 0,
146
+ }));
147
+
148
+ const allActive = this.store.getEngramsByAgent(
149
+ query.agentId,
150
+ query.includeStaging ? undefined : 'active',
151
+ query.includeRetracted ?? false
152
+ );
153
+
154
+ // Merge candidates (deduplicate)
155
+ const candidateMap = new Map<string, Engram>();
156
+ for (const r of bm25Ranked) candidateMap.set(r.engram.id, r.engram);
157
+ for (const e of allActive) candidateMap.set(e.id, e);
158
+ const candidates = Array.from(candidateMap.values());
159
+
160
+ if (candidates.length === 0) return [];
161
+
162
+ // Tokenize query once
163
+ const queryTokens = tokenize(query.context);
164
+
165
+ // Phase 3a: Compute raw cosine similarities for adaptive normalization
166
+ const rawCosineSims = new Map<string, number>();
167
+ if (queryEmbedding) {
168
+ for (const engram of candidates) {
169
+ if (engram.embedding) {
170
+ rawCosineSims.set(engram.id, cosineSimilarity(queryEmbedding, engram.embedding));
171
+ }
172
+ }
173
+ }
174
+
175
+ // Compute distribution stats for model-agnostic normalization
176
+ const simValues = Array.from(rawCosineSims.values());
177
+ const simMean = simValues.length > 0
178
+ ? simValues.reduce((a, b) => a + b, 0) / simValues.length : 0;
179
+ const rawStdDev = simValues.length > 1
180
+ ? Math.sqrt(simValues.reduce((sum, s) => sum + (s - simMean) ** 2, 0) / simValues.length) : 0.15;
181
+ // Floor stddev at 0.10 to prevent z-score inflation with small candidate pools
182
+ const simStdDev = Math.max(rawStdDev, 0.10);
183
+
184
+ // Phase 3b: Score each candidate with per-phase breakdown
185
+ const scored = candidates.map(engram => {
186
+ const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
187
+ const associations = this.store.getAssociationsFor(engram.id);
188
+
189
+ // --- Text relevance (keyword signals) ---
190
+
191
+ // Signal 1: BM25 continuous score (0-1, from FTS5 rank)
192
+ const bm25Score = bm25ScoreMap.get(engram.id) ?? 0;
193
+
194
+ // Signal 2: Jaccard similarity with stopword filtering
195
+ const conceptTokens = tokenize(engram.concept);
196
+ const contentTokens = tokenize(engram.content);
197
+ const conceptJaccard = jaccard(queryTokens, conceptTokens);
198
+ const contentJaccard = jaccard(queryTokens, contentTokens);
199
+ const jaccardScore = 0.6 * conceptJaccard + 0.4 * contentJaccard;
200
+
201
+ // Signal 3: Concept exact match bonus (up to 0.3)
202
+ const conceptOverlap = conceptTokens.size > 0
203
+ ? [...conceptTokens].filter(w => queryTokens.has(w)).length / conceptTokens.size
204
+ : 0;
205
+ const conceptBonus = conceptOverlap * 0.3;
206
+
207
+ const keywordMatch = Math.min(Math.max(bm25Score, jaccardScore) + conceptBonus, 1.0);
208
+
209
+ // --- Vector similarity (semantic signal) ---
210
+ // Two-stage: absolute floor prevents noise, then z-score ranks within matches.
211
+ // Stage 1: Raw cosine must exceed mean + 1 stddev (absolute relevance gate)
212
+ // Stage 2: Z-score maps relative position to 0-1 for ranking quality
213
+ let vectorMatch = 0;
214
+ const rawSim = rawCosineSims.get(engram.id);
215
+ if (rawSim !== undefined) {
216
+ const zScore = (rawSim - simMean) / simStdDev;
217
+ // Gate: must be at least 1 stddev above mean to be considered a match
218
+ if (zScore > 1.0) {
219
+ // Map z=1..3 → 0..1 linearly
220
+ vectorMatch = Math.min(1, (zScore - 1.0) / 2.0);
221
+ }
222
+ }
223
+
224
+ // Combined text match: best of keyword and vector signals
225
+ const textMatch = Math.max(keywordMatch, vectorMatch);
226
+
227
+ // --- Temporal signals ---
228
+
229
+ // ACT-R decay — confidence + replay modulated (synaptic tagging)
230
+ // High-confidence memories decay slower. Heavily-accessed memories also resist decay.
231
+ // Default exponent: 0.5. High confidence (0.8+): 0.3. High access (10+): further -0.05.
232
+ const confMod = 0.2 * Math.max(0, (engram.confidence - 0.5) / 0.5);
233
+ const replayMod = Math.min(0.1, 0.05 * Math.log1p(engram.accessCount));
234
+ const decayExponent = Math.max(0.2, 0.5 - confMod - replayMod);
235
+ const decayScore = baseLevelActivation(engram.accessCount, ageDays, decayExponent);
236
+
237
+ // Hebbian boost from associations — capped to prevent popular memories
238
+ // from dominating regardless of query relevance
239
+ const rawHebbian = associations.length > 0
240
+ ? associations.reduce((sum, a) => sum + a.weight, 0) / associations.length
241
+ : 0;
242
+ const hebbianBoost = Math.min(rawHebbian, 0.5);
243
+
244
+ // Centrality signal — well-connected memories (high weighted degree)
245
+ // get a small boost. This makes consolidation edges matter for retrieval.
246
+ // Log-scaled to prevent hub domination: 10 edges ≈ 0.05 boost, 50 ≈ 0.08
247
+ const weightedDegree = associations.reduce((sum, a) => sum + a.weight, 0);
248
+ const centralityBoost = associations.length > 0
249
+ ? Math.min(0.1, 0.03 * Math.log1p(weightedDegree))
250
+ : 0;
251
+
252
+ // Confidence gate — multiplicative quality signal
253
+ const confidenceGate = engram.confidence;
254
+
255
+ // Feedback bonus — memories confirmed useful via explicit feedback get a
256
+ // direct additive boost. Models how a senior dev "just knows" certain things
257
+ // are important. Confidence > 0.6 means at least 2+ positive feedbacks.
258
+ // Scales: conf 0.6→0.03, 0.7→0.06, 0.8→0.09, 1.0→0.15
259
+ const feedbackBonus = engram.confidence > 0.55
260
+ ? Math.min(0.15, 0.3 * Math.max(0, engram.confidence - 0.5))
261
+ : 0;
262
+
263
+ // --- Composite score: relevance-gated additive ---
264
+ // Text relevance must be present for temporal signals to contribute.
265
+ // Without text relevance, a memory shouldn't activate regardless of recency.
266
+ // Temporal contribution scales with text relevance (weak match = weak temporal boost).
267
+ const temporalNorm = Math.min(softplus(decayScore + hebbianBoost), 3.0) / 3.0;
268
+ const relevanceGate = textMatch > 0.1 ? textMatch : 0.0; // Proportional gate
269
+ const composite = (0.6 * textMatch + 0.4 * temporalNorm * relevanceGate + centralityBoost * relevanceGate + feedbackBonus * relevanceGate) * confidenceGate;
270
+
271
+ const phaseScores: PhaseScores = {
272
+ textMatch,
273
+ vectorMatch,
274
+ decayScore,
275
+ hebbianBoost,
276
+ graphBoost: 0, // Filled in phase 5
277
+ confidenceGate,
278
+ composite,
279
+ rerankerScore: 0, // Filled in phase 7
280
+ };
281
+
282
+ return { engram, score: composite, phaseScores, associations };
283
+ });
284
+
285
+ // Phase 3.5: Rocchio pseudo-relevance feedback — expand query with top result terms
286
+ // then re-search BM25 to find candidates that keyword search missed
287
+ const preSorted = scored.sort((a, b) => b.score - a.score);
288
+ const topForFeedback = preSorted.slice(0, 3).filter(r => r.phaseScores.textMatch > 0.1);
289
+ if (topForFeedback.length > 0) {
290
+ const feedbackTerms = new Set<string>();
291
+ for (const item of topForFeedback) {
292
+ const tokens = tokenize(item.engram.content);
293
+ for (const t of tokens) {
294
+ if (!queryTokens.has(t) && t.length >= 4) feedbackTerms.add(t);
295
+ }
296
+ }
297
+ // Take top 5 feedback terms and re-search
298
+ const extraTerms = Array.from(feedbackTerms).slice(0, 5).join(' ');
299
+ if (extraTerms) {
300
+ const feedbackBM25 = this.store.searchBM25WithRank(query.agentId, `${searchContext} ${extraTerms}`, limit * 2);
301
+ for (const r of feedbackBM25) {
302
+ if (!candidateMap.has(r.engram.id)) {
303
+ candidateMap.set(r.engram.id, r.engram);
304
+ // Score the new candidate
305
+ const engram = r.engram;
306
+ const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
307
+ const associations = this.store.getAssociationsFor(engram.id);
308
+ const cTokens = tokenize(engram.concept);
309
+ const ctTokens = tokenize(engram.content);
310
+ const cJac = jaccard(queryTokens, cTokens);
311
+ const ctJac = jaccard(queryTokens, ctTokens);
312
+ const jSc = 0.6 * cJac + 0.4 * ctJac;
313
+ const cOvlp = cTokens.size > 0 ? [...cTokens].filter(w => queryTokens.has(w)).length / cTokens.size : 0;
314
+ const km = Math.min(Math.max(r.bm25Score, jSc) + cOvlp * 0.3, 1.0);
315
+ let vm = 0;
316
+ const rs = rawCosineSims.get(engram.id) ?? (queryEmbedding && engram.embedding ? cosineSimilarity(queryEmbedding, engram.embedding) : 0);
317
+ if (rs) {
318
+ const z = (rs - simMean) / simStdDev;
319
+ if (z > 1.0) vm = Math.min(1, (z - 1.0) / 2.0);
320
+ }
321
+ const tm = Math.max(km, vm);
322
+ const ds = baseLevelActivation(engram.accessCount, ageDays);
323
+ const rh = associations.length > 0 ? Math.min(associations.reduce((s, a) => s + a.weight, 0) / associations.length, 0.5) : 0;
324
+ const tn = Math.min(softplus(ds + rh), 3.0) / 3.0;
325
+ const rg = tm > 0.1 ? tm : 0.0;
326
+ const comp = (0.6 * tm + 0.4 * tn * rg) * engram.confidence;
327
+ scored.push({ engram, score: comp, phaseScores: { textMatch: tm, vectorMatch: vm, decayScore: ds, hebbianBoost: rh, graphBoost: 0, confidenceGate: engram.confidence, composite: comp, rerankerScore: 0 }, associations });
328
+ }
329
+ }
330
+ }
331
+ }
332
+
333
+ // Phase 3.7: Entity-Bridge boost — boost scored candidates that share entity tags
334
+ // with the most query-relevant result. Only bridge from the single best text-match
335
+ // to avoid pulling in unrelated entities from tangentially-matching results.
336
+ {
337
+ // Find the result with the highest textMatch (most query-relevant, not just highest score)
338
+ // Gate: only bridge when anchor has meaningful text relevance (> 0.15)
339
+ // Adaptive: scale bridge boost inversely with candidate pool size to prevent
340
+ // over-boosting in large memory pools where many items share entity tags
341
+ const sortedByTextMatch = scored
342
+ .filter(r => r.phaseScores.textMatch > 0.15)
343
+ .sort((a, b) => b.phaseScores.textMatch - a.phaseScores.textMatch);
344
+
345
+ // Bridge from top 2 text-matched results (IDF handles weighting)
346
+ const bridgeAnchors = sortedByTextMatch.slice(0, 2);
347
+
348
+ if (bridgeAnchors.length > 0) {
349
+ const entityTags = new Set<string>();
350
+ const anchorIds = new Set(bridgeAnchors.map(r => r.engram.id));
351
+
352
+ for (const item of bridgeAnchors) {
353
+ for (const tag of item.engram.tags) {
354
+ const t = tag.toLowerCase();
355
+ // Skip non-entity tags: turn IDs, session tags, dialogue IDs, generic speaker labels
356
+ if (/^t\d+$/.test(t) || t.startsWith('session-') || t.startsWith('dia_') || t.length < 3) continue;
357
+ if (/^speaker\d*$/.test(t)) continue; // Generic speaker labels are too broad
358
+ entityTags.add(t);
359
+ }
360
+ }
361
+
362
+ // Document frequency filter: remove tags appearing in >30% of items (too common)
363
+ // This prevents speaker names in 2-person conversations from being used as bridges
364
+ if (entityTags.size > 0 && scored.length > 10) {
365
+ const tagFreqs = new Map<string, number>();
366
+ for (const item of scored) {
367
+ const seen = new Set<string>();
368
+ for (const tag of item.engram.tags) {
369
+ const t = tag.toLowerCase();
370
+ if (entityTags.has(t) && !seen.has(t)) {
371
+ seen.add(t);
372
+ tagFreqs.set(t, (tagFreqs.get(t) ?? 0) + 1);
373
+ }
374
+ }
375
+ }
376
+ const maxFreq = scored.length * 0.30;
377
+ for (const [tag, freq] of tagFreqs) {
378
+ if (freq > maxFreq) entityTags.delete(tag);
379
+ }
380
+ }
381
+
382
+ if (entityTags.size > 0) {
383
+ for (const item of scored) {
384
+ if (anchorIds.has(item.engram.id)) continue;
385
+
386
+ const engramTags = new Set(item.engram.tags.map((t: string) => t.toLowerCase()));
387
+ let sharedEntities = 0;
388
+ for (const et of entityTags) {
389
+ if (engramTags.has(et)) sharedEntities++;
390
+ }
391
+
392
+ if (sharedEntities > 0) {
393
+ // Flat bridge boost per shared entity
394
+ const bridgeBoost = Math.min(sharedEntities * 0.15, 0.4);
395
+ item.score += bridgeBoost;
396
+ item.phaseScores.composite += bridgeBoost;
397
+ item.phaseScores.graphBoost += bridgeBoost;
398
+ }
399
+ }
400
+ }
401
+ }
402
+ }
403
+
404
+ // Phase 4+5: Graph walk — boost engrams connected to high-scoring ones
405
+ // Only walk from engrams that had text relevance (composite > 0 pre-walk)
406
+ const sorted = scored.sort((a, b) => b.score - a.score);
407
+ const topN = sorted.slice(0, limit * 3);
408
+ this.graphWalk(topN, 2, 0.3);
409
+
410
+ // Phase 6: Initial filter and sort for re-ranking pool
411
+ const pool = topN
412
+ .filter(r => r.score >= minScore)
413
+ .sort((a, b) => b.score - a.score);
414
+
415
+ // Phase 7: Cross-encoder re-ranking — scores (query, passage) pairs directly
416
+ // Widens the pool to find relevant results that keyword matching missed
417
+ const rerankPool = pool.slice(0, Math.max(limit * 3, 30));
418
+
419
+ if (useReranker && rerankPool.length > 0) {
420
+ try {
421
+ const passages = rerankPool.map(r =>
422
+ `${r.engram.concept}: ${r.engram.content}`
423
+ );
424
+ const rerankResults = await rerank(query.context, passages);
425
+
426
+ // Adaptive reranker blend (Codex recommendation):
427
+ // When BM25/text signals are strong, trust them more; when weak, lean on reranker.
428
+ const bm25Max = Math.max(...rerankPool.map(r => r.phaseScores.textMatch));
429
+ const rerankWeight = Math.min(0.7, Math.max(0.3, 0.3 + 0.4 * (1 - bm25Max)));
430
+ const compositeWeight = 1 - rerankWeight;
431
+
432
+ for (const rr of rerankResults) {
433
+ const item = rerankPool[rr.index];
434
+ item.phaseScores.rerankerScore = rr.score;
435
+ item.score = compositeWeight * item.phaseScores.composite + rerankWeight * rr.score;
436
+ }
437
+ } catch {
438
+ // Re-ranker unavailable — keep original scores
439
+ }
440
+ }
441
+
442
+ // Phase 8: Multi-channel OOD detection + agreement gate
443
+ // Requires at least 2 of 3 retrieval channels to agree the query is in-domain.
444
+ if (rerankPool.length >= 3) {
445
+ const topBM25 = Math.max(...rerankPool.map(r => bm25ScoreMap.get(r.engram.id) ?? 0));
446
+ const topVector = queryEmbedding
447
+ ? Math.max(...rerankPool.map(r => r.phaseScores.vectorMatch))
448
+ : 0;
449
+ const topReranker = Math.max(...rerankPool.map(r => r.phaseScores.rerankerScore));
450
+
451
+ const bm25Ok = topBM25 > 0.3;
452
+ const vectorOk = topVector > 0.05;
453
+ const rerankerOk = topReranker > 0.25;
454
+ const channelsAgreeing = (bm25Ok ? 1 : 0) + (vectorOk ? 1 : 0) + (rerankerOk ? 1 : 0);
455
+
456
+ const rerankerScores = rerankPool
457
+ .map(r => r.phaseScores.rerankerScore)
458
+ .sort((a, b) => b - a);
459
+ const margin = rerankerScores.length >= 2
460
+ ? rerankerScores[0] - rerankerScores[1]
461
+ : rerankerScores[0];
462
+
463
+ const maxRawCosine = queryEmbedding && simValues.length > 0
464
+ ? Math.max(...simValues)
465
+ : 1.0;
466
+
467
+ // Stricter gate when caller explicitly requests abstention (e.g., noise filter queries)
468
+ const requiredChannels = abstentionThreshold > 0 ? 3 : 2;
469
+
470
+ // Hard abstention: fewer than required channels agree AND semantic drift is high
471
+ if (channelsAgreeing < requiredChannels && maxRawCosine < (simMean + simStdDev * 1.5)) {
472
+ return [];
473
+ }
474
+
475
+ // Soft penalty: only 1 channel agrees or margin is thin
476
+ if (channelsAgreeing < 2 || margin < 0.05) {
477
+ // If caller explicitly requested abstention, honor it when agreement is weak
478
+ if (abstentionThreshold > 0) {
479
+ return [];
480
+ }
481
+ for (const item of rerankPool) {
482
+ item.score *= 0.4;
483
+ }
484
+ }
485
+ }
486
+
487
+ // Legacy abstention gate (when explicitly requested)
488
+ if (abstentionThreshold > 0 && rerankPool.length >= 3) {
489
+ const topRerankerScores = rerankPool
490
+ .map(r => r.phaseScores.rerankerScore)
491
+ .sort((a, b) => b - a)
492
+ .slice(0, 5);
493
+ const maxScore = topRerankerScores[0];
494
+ const meanScore = topRerankerScores.reduce((s, v) => s + v, 0) / topRerankerScores.length;
495
+ const variance = topRerankerScores.reduce((s, v) => s + (v - meanScore) ** 2, 0) / topRerankerScores.length;
496
+
497
+ if (maxScore < abstentionThreshold || (maxScore < 0.5 && variance < 0.01)) {
498
+ return [];
499
+ }
500
+ }
501
+
502
+ // Phase 8c: Supersession penalty — superseded memories are deprioritized.
503
+ // They aren't wrong (that's retraction), just outdated.
504
+ for (const item of rerankPool) {
505
+ if (item.engram.supersededBy) {
506
+ item.score *= 0.15; // Severe down-rank — successor should dominate
507
+ }
508
+ }
509
+
510
+ // Phase 9: Final sort, limit, explain
511
+ const results: ActivationResult[] = rerankPool
512
+ .sort((a, b) => b.score - a.score)
513
+ .slice(0, limit)
514
+ .map(r => ({
515
+ engram: r.engram,
516
+ score: r.score,
517
+ phaseScores: r.phaseScores,
518
+ why: this.explain(r.phaseScores, r.engram, r.associations),
519
+ associations: r.associations,
520
+ }));
521
+
522
+ const activatedIds = results.map(r => r.engram.id);
523
+
524
+ // Side effects: touch, co-activate, Hebbian update (skip for internal/system calls)
525
+ if (!query.internal) {
526
+ for (const id of activatedIds) {
527
+ this.store.touchEngram(id);
528
+ }
529
+ this.coActivationBuffer.pushBatch(activatedIds);
530
+ this.updateHebbianWeights();
531
+
532
+ // Log activation event for eval
533
+ const latencyMs = performance.now() - startTime;
534
+ this.store.logActivationEvent({
535
+ id: randomUUID(),
536
+ agentId: query.agentId,
537
+ timestamp: new Date(),
538
+ context: query.context,
539
+ resultsReturned: results.length,
540
+ topScore: results.length > 0 ? results[0].score : 0,
541
+ latencyMs,
542
+ engramIds: activatedIds,
543
+ });
544
+ }
545
+
546
+ return results;
547
+ }
548
+
549
+ /**
550
+ * Beam search graph walk — replaces naive BFS.
551
+ * Scores paths (not just nodes), uses query-dependent edge filtering,
552
+ * and supports deeper exploration with focused beams.
553
+ */
554
+ private graphWalk(
555
+ scored: { engram: Engram; score: number; phaseScores: PhaseScores; associations: Association[] }[],
556
+ maxDepth: number,
557
+ hopPenalty: number
558
+ ): void {
559
+ const scoreMap = new Map(scored.map(s => [s.engram.id, s]));
560
+ const MAX_TOTAL_BOOST = 0.25; // Slightly higher cap for beam search (deeper paths earn it)
561
+ const BEAM_WIDTH = 15;
562
+
563
+ // Seed the beam with high-scoring, text-relevant items
564
+ const beam = scored
565
+ .filter(item => item.phaseScores.textMatch >= 0.15)
566
+ .sort((a, b) => b.score - a.score)
567
+ .slice(0, BEAM_WIDTH);
568
+
569
+ // Track which engrams have been explored (avoid cycles)
570
+ const explored = new Set<string>();
571
+
572
+ for (let depth = 0; depth < maxDepth; depth++) {
573
+ const nextBeam: typeof beam = [];
574
+
575
+ for (const item of beam) {
576
+ if (explored.has(item.engram.id)) continue;
577
+ explored.add(item.engram.id);
578
+
579
+ // Get associations — for depth > 0, fetch from store if not in scored set
580
+ const associations = item.associations.length > 0
581
+ ? item.associations
582
+ : this.store.getAssociationsFor(item.engram.id);
583
+
584
+ for (const assoc of associations) {
585
+ const neighborId = assoc.fromEngramId === item.engram.id
586
+ ? assoc.toEngramId
587
+ : assoc.fromEngramId;
588
+
589
+ if (explored.has(neighborId)) continue;
590
+
591
+ const neighbor = scoreMap.get(neighborId);
592
+ if (!neighbor) continue;
593
+
594
+ // Query-dependent edge filtering: neighbor must have SOME relevance
595
+ // (textMatch > 0.05 for deeper hops, relaxed from 0.1)
596
+ const relevanceFloor = depth === 0 ? 0.1 : 0.05;
597
+ if (neighbor.phaseScores.textMatch < relevanceFloor) continue;
598
+
599
+ // Skip if neighbor already at boost cap
600
+ if (neighbor.phaseScores.graphBoost >= MAX_TOTAL_BOOST) continue;
601
+
602
+ // Path score: source score * edge weight * hop penalty^(depth+1)
603
+ const normalizedWeight = Math.min(assoc.weight, 5.0) / 5.0;
604
+ const pathScore = item.score * normalizedWeight * Math.pow(hopPenalty, depth + 1);
605
+
606
+ const boost = Math.min(pathScore, 0.15, MAX_TOTAL_BOOST - neighbor.phaseScores.graphBoost);
607
+ if (boost > 0.001) {
608
+ neighbor.score += boost;
609
+ neighbor.phaseScores.graphBoost += boost;
610
+ nextBeam.push(neighbor);
611
+ }
612
+ }
613
+ }
614
+
615
+ // Prune beam for next depth level
616
+ if (nextBeam.length === 0) break;
617
+ beam.length = 0;
618
+ beam.push(...nextBeam
619
+ .sort((a, b) => b.score - a.score)
620
+ .slice(0, BEAM_WIDTH)
621
+ );
622
+ }
623
+ }
624
+
625
+ private updateHebbianWeights(): void {
626
+ const pairs = this.coActivationBuffer.getCoActivatedPairs(10_000);
627
+ // Deduplicate pairs to prevent repeated strengthening
628
+ const seen = new Set<string>();
629
+ for (const [a, b] of pairs) {
630
+ const key = a < b ? `${a}:${b}` : `${b}:${a}`;
631
+ if (seen.has(key)) continue;
632
+ seen.add(key);
633
+
634
+ const existing = this.store.getAssociation(a, b) ?? this.store.getAssociation(b, a);
635
+ const currentWeight = existing?.weight ?? 0.1;
636
+ const newWeight = strengthenAssociation(currentWeight);
637
+ this.store.upsertAssociation(a, b, newWeight, 'hebbian');
638
+ this.store.upsertAssociation(b, a, newWeight, 'hebbian');
639
+ }
640
+ }
641
+
642
+ private explain(phases: PhaseScores, engram: Engram, associations: Association[]): string {
643
+ const parts: string[] = [];
644
+ parts.push(`composite=${phases.composite.toFixed(3)}`);
645
+ if (phases.textMatch > 0) parts.push(`text=${phases.textMatch.toFixed(2)}`);
646
+ if (phases.vectorMatch > 0) parts.push(`vector=${phases.vectorMatch.toFixed(2)}`);
647
+ parts.push(`decay=${phases.decayScore.toFixed(2)}`);
648
+ if (phases.hebbianBoost > 0) parts.push(`hebbian=${phases.hebbianBoost.toFixed(2)}`);
649
+ if (phases.graphBoost > 0) parts.push(`graph=${phases.graphBoost.toFixed(2)}`);
650
+ if (phases.rerankerScore > 0) parts.push(`reranker=${phases.rerankerScore.toFixed(2)}`);
651
+ parts.push(`conf=${phases.confidenceGate.toFixed(2)}`);
652
+ parts.push(`access=${engram.accessCount}`);
653
+ if (associations.length > 0) parts.push(`edges=${associations.length}`);
654
+ return parts.join(' | ');
655
+ }
656
+ }