agent-working-memory 0.5.4 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -46
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +21 -5
- package/dist/api/routes.js.map +1 -1
- package/dist/cli.js +67 -67
- package/dist/coordination/index.d.ts +11 -0
- package/dist/coordination/index.d.ts.map +1 -0
- package/dist/coordination/index.js +39 -0
- package/dist/coordination/index.js.map +1 -0
- package/dist/coordination/mcp-tools.d.ts +8 -0
- package/dist/coordination/mcp-tools.d.ts.map +1 -0
- package/dist/coordination/mcp-tools.js +216 -0
- package/dist/coordination/mcp-tools.js.map +1 -0
- package/dist/coordination/routes.d.ts +9 -0
- package/dist/coordination/routes.d.ts.map +1 -0
- package/dist/coordination/routes.js +434 -0
- package/dist/coordination/routes.js.map +1 -0
- package/dist/coordination/schema.d.ts +12 -0
- package/dist/coordination/schema.d.ts.map +1 -0
- package/dist/coordination/schema.js +91 -0
- package/dist/coordination/schema.js.map +1 -0
- package/dist/coordination/schemas.d.ts +208 -0
- package/dist/coordination/schemas.d.ts.map +1 -0
- package/dist/coordination/schemas.js +109 -0
- package/dist/coordination/schemas.js.map +1 -0
- package/dist/coordination/stale.d.ts +25 -0
- package/dist/coordination/stale.d.ts.map +1 -0
- package/dist/coordination/stale.js +53 -0
- package/dist/coordination/stale.js.map +1 -0
- package/dist/index.js +21 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +90 -79
- package/dist/mcp.js.map +1 -1
- package/dist/storage/sqlite.d.ts +3 -0
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +285 -281
- package/dist/storage/sqlite.js.map +1 -1
- package/package.json +55 -55
- package/src/api/index.ts +3 -3
- package/src/api/routes.ts +551 -536
- package/src/cli.ts +397 -397
- package/src/coordination/index.ts +47 -0
- package/src/coordination/mcp-tools.ts +313 -0
- package/src/coordination/routes.ts +656 -0
- package/src/coordination/schema.ts +94 -0
- package/src/coordination/schemas.ts +136 -0
- package/src/coordination/stale.ts +89 -0
- package/src/core/decay.ts +63 -63
- package/src/core/embeddings.ts +88 -88
- package/src/core/hebbian.ts +93 -93
- package/src/core/index.ts +5 -5
- package/src/core/logger.ts +36 -36
- package/src/core/query-expander.ts +66 -66
- package/src/core/reranker.ts +101 -101
- package/src/engine/activation.ts +656 -656
- package/src/engine/connections.ts +103 -103
- package/src/engine/consolidation-scheduler.ts +125 -125
- package/src/engine/eval.ts +102 -102
- package/src/engine/eviction.ts +101 -101
- package/src/engine/index.ts +8 -8
- package/src/engine/retraction.ts +100 -100
- package/src/engine/staging.ts +74 -74
- package/src/index.ts +137 -121
- package/src/mcp.ts +1024 -1013
- package/src/storage/index.ts +3 -3
- package/src/storage/sqlite.ts +968 -963
- package/src/types/agent.ts +67 -67
- package/src/types/checkpoint.ts +46 -46
- package/src/types/engram.ts +217 -217
- package/src/types/eval.ts +100 -100
- package/src/types/index.ts +6 -6
package/src/engine/activation.ts
CHANGED
|
@@ -1,656 +1,656 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
/**
|
|
4
|
-
* Activation Pipeline — the core retrieval engine.
|
|
5
|
-
*
|
|
6
|
-
* 10-phase cognitive retrieval pipeline:
|
|
7
|
-
* 0. Query expansion (flan-t5-small synonym generation)
|
|
8
|
-
* 1. Vector embedding (MiniLM 384d)
|
|
9
|
-
* 2. Parallel retrieval (FTS5/BM25 + vector pool)
|
|
10
|
-
* 3. Scoring (BM25, Jaccard, z-score vector, entity-bridge boost)
|
|
11
|
-
* 4. Rocchio pseudo-relevance feedback (expand + re-search BM25)
|
|
12
|
-
* 5. ACT-R temporal decay
|
|
13
|
-
* 6. Hebbian boost (co-activation strength)
|
|
14
|
-
* 7. Composite scoring with confidence gating
|
|
15
|
-
* 8. Beam search graph walk (depth 2, hop penalty)
|
|
16
|
-
* 9. Cross-encoder reranking (ms-marco-MiniLM, adaptive blend)
|
|
17
|
-
* 10. Abstention gate
|
|
18
|
-
*
|
|
19
|
-
* Logs activation events for eval metrics.
|
|
20
|
-
*/
|
|
21
|
-
|
|
22
|
-
import { randomUUID } from 'node:crypto';
|
|
23
|
-
import { baseLevelActivation, softplus } from '../core/decay.js';
|
|
24
|
-
import { strengthenAssociation, CoActivationBuffer } from '../core/hebbian.js';
|
|
25
|
-
import { embed, cosineSimilarity } from '../core/embeddings.js';
|
|
26
|
-
import { rerank } from '../core/reranker.js';
|
|
27
|
-
import { expandQuery } from '../core/query-expander.js';
|
|
28
|
-
import type {
|
|
29
|
-
Engram, ActivationResult, ActivationQuery, Association, PhaseScores,
|
|
30
|
-
} from '../types/index.js';
|
|
31
|
-
import type { EngramStore } from '../storage/sqlite.js';
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Common English stopwords — filtered from similarity calculations.
|
|
35
|
-
* These words carry no semantic signal for memory retrieval.
|
|
36
|
-
*/
|
|
37
|
-
const STOPWORDS = new Set([
|
|
38
|
-
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
|
|
39
|
-
'her', 'was', 'one', 'our', 'out', 'has', 'have', 'been', 'from', 'that',
|
|
40
|
-
'this', 'with', 'they', 'will', 'each', 'make', 'like', 'then', 'than',
|
|
41
|
-
'them', 'some', 'what', 'when', 'where', 'which', 'who', 'how', 'use',
|
|
42
|
-
'into', 'does', 'also', 'just', 'more', 'over', 'such', 'only', 'very',
|
|
43
|
-
'about', 'after', 'being', 'between', 'could', 'during', 'before',
|
|
44
|
-
'should', 'would', 'their', 'there', 'these', 'those', 'through',
|
|
45
|
-
'because', 'using', 'other',
|
|
46
|
-
]);
|
|
47
|
-
|
|
48
|
-
function tokenize(text: string): Set<string> {
|
|
49
|
-
return new Set(
|
|
50
|
-
text.toLowerCase()
|
|
51
|
-
.split(/\s+/)
|
|
52
|
-
.filter(w => w.length > 2 && !STOPWORDS.has(w))
|
|
53
|
-
);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Jaccard similarity between two word sets: |intersection| / |union|
|
|
58
|
-
*/
|
|
59
|
-
function jaccard(a: Set<string>, b: Set<string>): number {
|
|
60
|
-
if (a.size === 0 || b.size === 0) return 0;
|
|
61
|
-
let intersection = 0;
|
|
62
|
-
for (const w of a) {
|
|
63
|
-
if (b.has(w)) intersection++;
|
|
64
|
-
}
|
|
65
|
-
const union = a.size + b.size - intersection;
|
|
66
|
-
return union > 0 ? intersection / union : 0;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export class ActivationEngine {
|
|
70
|
-
private store: EngramStore;
|
|
71
|
-
private coActivationBuffer: CoActivationBuffer;
|
|
72
|
-
|
|
73
|
-
constructor(store: EngramStore) {
|
|
74
|
-
this.store = store;
|
|
75
|
-
this.coActivationBuffer = new CoActivationBuffer(50);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Activate — retrieve the most cognitively relevant engrams for a context.
|
|
80
|
-
*/
|
|
81
|
-
async activate(query: ActivationQuery): Promise<ActivationResult[]> {
|
|
82
|
-
const startTime = performance.now();
|
|
83
|
-
const limit = query.limit ?? 10;
|
|
84
|
-
const minScore = query.minScore ?? 0.01; // Default: filter out zero-relevance results
|
|
85
|
-
const useReranker = query.useReranker ?? true;
|
|
86
|
-
const useExpansion = query.useExpansion ?? true;
|
|
87
|
-
const abstentionThreshold = query.abstentionThreshold ?? 0;
|
|
88
|
-
|
|
89
|
-
// Phase -1: Coref expansion — if query has pronouns, append recent entity names
|
|
90
|
-
// Helps conversational recall where "she/he/they/it" refers to a named entity.
|
|
91
|
-
let queryContext = query.context;
|
|
92
|
-
const pronounPattern = /\b(she|he|they|her|his|him|their|it|that|this|there)\b/i;
|
|
93
|
-
if (pronounPattern.test(queryContext)) {
|
|
94
|
-
// Pull recent entity names from the agent's most-accessed memories
|
|
95
|
-
try {
|
|
96
|
-
const recentEntities = this.store.getEngramsByAgent(query.agentId, 'active')
|
|
97
|
-
.sort((a, b) => b.accessCount - a.accessCount)
|
|
98
|
-
.slice(0, 10)
|
|
99
|
-
.flatMap(e => e.tags.filter(t => t.length >= 3 && !/^(session-|low-|D\d)/.test(t)))
|
|
100
|
-
.filter((v, i, a) => a.indexOf(v) === i)
|
|
101
|
-
.slice(0, 5);
|
|
102
|
-
if (recentEntities.length > 0) {
|
|
103
|
-
queryContext = `${queryContext} ${recentEntities.join(' ')}`;
|
|
104
|
-
}
|
|
105
|
-
} catch { /* non-fatal */ }
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// Phase 0: Query expansion — add related terms to improve BM25 recall
|
|
109
|
-
let searchContext = queryContext;
|
|
110
|
-
if (useExpansion) {
|
|
111
|
-
try {
|
|
112
|
-
searchContext = await expandQuery(query.context);
|
|
113
|
-
} catch {
|
|
114
|
-
// Expansion unavailable — use original query
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// Phase 1: Embed query for vector similarity (uses coref-expanded context)
|
|
119
|
-
let queryEmbedding: number[] | null = null;
|
|
120
|
-
try {
|
|
121
|
-
queryEmbedding = await embed(queryContext);
|
|
122
|
-
} catch {
|
|
123
|
-
// Embedding unavailable — fall back to text-only matching
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Phase 2: Parallel retrieval — dual BM25 + all active engrams
|
|
127
|
-
// Two-pass BM25: (1) keyword-stripped query for precision, (2) expanded query for recall.
|
|
128
|
-
const keywordQuery = Array.from(tokenize(query.context)).join(' ');
|
|
129
|
-
const bm25Keyword = keywordQuery.length > 2
|
|
130
|
-
? this.store.searchBM25WithRank(query.agentId, keywordQuery, limit * 3)
|
|
131
|
-
: [];
|
|
132
|
-
const bm25Expanded = this.store.searchBM25WithRank(query.agentId, searchContext, limit * 3);
|
|
133
|
-
|
|
134
|
-
// Merge: take the best BM25 score per engram from either pass
|
|
135
|
-
const bm25ScoreMap = new Map<string, number>();
|
|
136
|
-
const bm25EngramMap = new Map<string, any>();
|
|
137
|
-
for (const r of [...bm25Keyword, ...bm25Expanded]) {
|
|
138
|
-
const existing = bm25ScoreMap.get(r.engram.id) ?? 0;
|
|
139
|
-
if (r.bm25Score > existing) {
|
|
140
|
-
bm25ScoreMap.set(r.engram.id, r.bm25Score);
|
|
141
|
-
bm25EngramMap.set(r.engram.id, r.engram);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
const bm25Ranked = Array.from(bm25EngramMap.entries()).map(([id, engram]) => ({
|
|
145
|
-
engram, bm25Score: bm25ScoreMap.get(id) ?? 0,
|
|
146
|
-
}));
|
|
147
|
-
|
|
148
|
-
const allActive = this.store.getEngramsByAgent(
|
|
149
|
-
query.agentId,
|
|
150
|
-
query.includeStaging ? undefined : 'active',
|
|
151
|
-
query.includeRetracted ?? false
|
|
152
|
-
);
|
|
153
|
-
|
|
154
|
-
// Merge candidates (deduplicate)
|
|
155
|
-
const candidateMap = new Map<string, Engram>();
|
|
156
|
-
for (const r of bm25Ranked) candidateMap.set(r.engram.id, r.engram);
|
|
157
|
-
for (const e of allActive) candidateMap.set(e.id, e);
|
|
158
|
-
const candidates = Array.from(candidateMap.values());
|
|
159
|
-
|
|
160
|
-
if (candidates.length === 0) return [];
|
|
161
|
-
|
|
162
|
-
// Tokenize query once
|
|
163
|
-
const queryTokens = tokenize(query.context);
|
|
164
|
-
|
|
165
|
-
// Phase 3a: Compute raw cosine similarities for adaptive normalization
|
|
166
|
-
const rawCosineSims = new Map<string, number>();
|
|
167
|
-
if (queryEmbedding) {
|
|
168
|
-
for (const engram of candidates) {
|
|
169
|
-
if (engram.embedding) {
|
|
170
|
-
rawCosineSims.set(engram.id, cosineSimilarity(queryEmbedding, engram.embedding));
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Compute distribution stats for model-agnostic normalization
|
|
176
|
-
const simValues = Array.from(rawCosineSims.values());
|
|
177
|
-
const simMean = simValues.length > 0
|
|
178
|
-
? simValues.reduce((a, b) => a + b, 0) / simValues.length : 0;
|
|
179
|
-
const rawStdDev = simValues.length > 1
|
|
180
|
-
? Math.sqrt(simValues.reduce((sum, s) => sum + (s - simMean) ** 2, 0) / simValues.length) : 0.15;
|
|
181
|
-
// Floor stddev at 0.10 to prevent z-score inflation with small candidate pools
|
|
182
|
-
const simStdDev = Math.max(rawStdDev, 0.10);
|
|
183
|
-
|
|
184
|
-
// Phase 3b: Score each candidate with per-phase breakdown
|
|
185
|
-
const scored = candidates.map(engram => {
|
|
186
|
-
const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
|
|
187
|
-
const associations = this.store.getAssociationsFor(engram.id);
|
|
188
|
-
|
|
189
|
-
// --- Text relevance (keyword signals) ---
|
|
190
|
-
|
|
191
|
-
// Signal 1: BM25 continuous score (0-1, from FTS5 rank)
|
|
192
|
-
const bm25Score = bm25ScoreMap.get(engram.id) ?? 0;
|
|
193
|
-
|
|
194
|
-
// Signal 2: Jaccard similarity with stopword filtering
|
|
195
|
-
const conceptTokens = tokenize(engram.concept);
|
|
196
|
-
const contentTokens = tokenize(engram.content);
|
|
197
|
-
const conceptJaccard = jaccard(queryTokens, conceptTokens);
|
|
198
|
-
const contentJaccard = jaccard(queryTokens, contentTokens);
|
|
199
|
-
const jaccardScore = 0.6 * conceptJaccard + 0.4 * contentJaccard;
|
|
200
|
-
|
|
201
|
-
// Signal 3: Concept exact match bonus (up to 0.3)
|
|
202
|
-
const conceptOverlap = conceptTokens.size > 0
|
|
203
|
-
? [...conceptTokens].filter(w => queryTokens.has(w)).length / conceptTokens.size
|
|
204
|
-
: 0;
|
|
205
|
-
const conceptBonus = conceptOverlap * 0.3;
|
|
206
|
-
|
|
207
|
-
const keywordMatch = Math.min(Math.max(bm25Score, jaccardScore) + conceptBonus, 1.0);
|
|
208
|
-
|
|
209
|
-
// --- Vector similarity (semantic signal) ---
|
|
210
|
-
// Two-stage: absolute floor prevents noise, then z-score ranks within matches.
|
|
211
|
-
// Stage 1: Raw cosine must exceed mean + 1 stddev (absolute relevance gate)
|
|
212
|
-
// Stage 2: Z-score maps relative position to 0-1 for ranking quality
|
|
213
|
-
let vectorMatch = 0;
|
|
214
|
-
const rawSim = rawCosineSims.get(engram.id);
|
|
215
|
-
if (rawSim !== undefined) {
|
|
216
|
-
const zScore = (rawSim - simMean) / simStdDev;
|
|
217
|
-
// Gate: must be at least 1 stddev above mean to be considered a match
|
|
218
|
-
if (zScore > 1.0) {
|
|
219
|
-
// Map z=1..3 → 0..1 linearly
|
|
220
|
-
vectorMatch = Math.min(1, (zScore - 1.0) / 2.0);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
// Combined text match: best of keyword and vector signals
|
|
225
|
-
const textMatch = Math.max(keywordMatch, vectorMatch);
|
|
226
|
-
|
|
227
|
-
// --- Temporal signals ---
|
|
228
|
-
|
|
229
|
-
// ACT-R decay — confidence + replay modulated (synaptic tagging)
|
|
230
|
-
// High-confidence memories decay slower. Heavily-accessed memories also resist decay.
|
|
231
|
-
// Default exponent: 0.5. High confidence (0.8+): 0.3. High access (10+): further -0.05.
|
|
232
|
-
const confMod = 0.2 * Math.max(0, (engram.confidence - 0.5) / 0.5);
|
|
233
|
-
const replayMod = Math.min(0.1, 0.05 * Math.log1p(engram.accessCount));
|
|
234
|
-
const decayExponent = Math.max(0.2, 0.5 - confMod - replayMod);
|
|
235
|
-
const decayScore = baseLevelActivation(engram.accessCount, ageDays, decayExponent);
|
|
236
|
-
|
|
237
|
-
// Hebbian boost from associations — capped to prevent popular memories
|
|
238
|
-
// from dominating regardless of query relevance
|
|
239
|
-
const rawHebbian = associations.length > 0
|
|
240
|
-
? associations.reduce((sum, a) => sum + a.weight, 0) / associations.length
|
|
241
|
-
: 0;
|
|
242
|
-
const hebbianBoost = Math.min(rawHebbian, 0.5);
|
|
243
|
-
|
|
244
|
-
// Centrality signal — well-connected memories (high weighted degree)
|
|
245
|
-
// get a small boost. This makes consolidation edges matter for retrieval.
|
|
246
|
-
// Log-scaled to prevent hub domination: 10 edges ≈ 0.05 boost, 50 ≈ 0.08
|
|
247
|
-
const weightedDegree = associations.reduce((sum, a) => sum + a.weight, 0);
|
|
248
|
-
const centralityBoost = associations.length > 0
|
|
249
|
-
? Math.min(0.1, 0.03 * Math.log1p(weightedDegree))
|
|
250
|
-
: 0;
|
|
251
|
-
|
|
252
|
-
// Confidence gate — multiplicative quality signal
|
|
253
|
-
const confidenceGate = engram.confidence;
|
|
254
|
-
|
|
255
|
-
// Feedback bonus — memories confirmed useful via explicit feedback get a
|
|
256
|
-
// direct additive boost. Models how a senior dev "just knows" certain things
|
|
257
|
-
// are important. Confidence > 0.6 means at least 2+ positive feedbacks.
|
|
258
|
-
// Scales: conf 0.6→0.03, 0.7→0.06, 0.8→0.09, 1.0→0.15
|
|
259
|
-
const feedbackBonus = engram.confidence > 0.55
|
|
260
|
-
? Math.min(0.15, 0.3 * Math.max(0, engram.confidence - 0.5))
|
|
261
|
-
: 0;
|
|
262
|
-
|
|
263
|
-
// --- Composite score: relevance-gated additive ---
|
|
264
|
-
// Text relevance must be present for temporal signals to contribute.
|
|
265
|
-
// Without text relevance, a memory shouldn't activate regardless of recency.
|
|
266
|
-
// Temporal contribution scales with text relevance (weak match = weak temporal boost).
|
|
267
|
-
const temporalNorm = Math.min(softplus(decayScore + hebbianBoost), 3.0) / 3.0;
|
|
268
|
-
const relevanceGate = textMatch > 0.1 ? textMatch : 0.0; // Proportional gate
|
|
269
|
-
const composite = (0.6 * textMatch + 0.4 * temporalNorm * relevanceGate + centralityBoost * relevanceGate + feedbackBonus * relevanceGate) * confidenceGate;
|
|
270
|
-
|
|
271
|
-
const phaseScores: PhaseScores = {
|
|
272
|
-
textMatch,
|
|
273
|
-
vectorMatch,
|
|
274
|
-
decayScore,
|
|
275
|
-
hebbianBoost,
|
|
276
|
-
graphBoost: 0, // Filled in phase 5
|
|
277
|
-
confidenceGate,
|
|
278
|
-
composite,
|
|
279
|
-
rerankerScore: 0, // Filled in phase 7
|
|
280
|
-
};
|
|
281
|
-
|
|
282
|
-
return { engram, score: composite, phaseScores, associations };
|
|
283
|
-
});
|
|
284
|
-
|
|
285
|
-
// Phase 3.5: Rocchio pseudo-relevance feedback — expand query with top result terms
|
|
286
|
-
// then re-search BM25 to find candidates that keyword search missed
|
|
287
|
-
const preSorted = scored.sort((a, b) => b.score - a.score);
|
|
288
|
-
const topForFeedback = preSorted.slice(0, 3).filter(r => r.phaseScores.textMatch > 0.1);
|
|
289
|
-
if (topForFeedback.length > 0) {
|
|
290
|
-
const feedbackTerms = new Set<string>();
|
|
291
|
-
for (const item of topForFeedback) {
|
|
292
|
-
const tokens = tokenize(item.engram.content);
|
|
293
|
-
for (const t of tokens) {
|
|
294
|
-
if (!queryTokens.has(t) && t.length >= 4) feedbackTerms.add(t);
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
// Take top 5 feedback terms and re-search
|
|
298
|
-
const extraTerms = Array.from(feedbackTerms).slice(0, 5).join(' ');
|
|
299
|
-
if (extraTerms) {
|
|
300
|
-
const feedbackBM25 = this.store.searchBM25WithRank(query.agentId, `${searchContext} ${extraTerms}`, limit * 2);
|
|
301
|
-
for (const r of feedbackBM25) {
|
|
302
|
-
if (!candidateMap.has(r.engram.id)) {
|
|
303
|
-
candidateMap.set(r.engram.id, r.engram);
|
|
304
|
-
// Score the new candidate
|
|
305
|
-
const engram = r.engram;
|
|
306
|
-
const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
|
|
307
|
-
const associations = this.store.getAssociationsFor(engram.id);
|
|
308
|
-
const cTokens = tokenize(engram.concept);
|
|
309
|
-
const ctTokens = tokenize(engram.content);
|
|
310
|
-
const cJac = jaccard(queryTokens, cTokens);
|
|
311
|
-
const ctJac = jaccard(queryTokens, ctTokens);
|
|
312
|
-
const jSc = 0.6 * cJac + 0.4 * ctJac;
|
|
313
|
-
const cOvlp = cTokens.size > 0 ? [...cTokens].filter(w => queryTokens.has(w)).length / cTokens.size : 0;
|
|
314
|
-
const km = Math.min(Math.max(r.bm25Score, jSc) + cOvlp * 0.3, 1.0);
|
|
315
|
-
let vm = 0;
|
|
316
|
-
const rs = rawCosineSims.get(engram.id) ?? (queryEmbedding && engram.embedding ? cosineSimilarity(queryEmbedding, engram.embedding) : 0);
|
|
317
|
-
if (rs) {
|
|
318
|
-
const z = (rs - simMean) / simStdDev;
|
|
319
|
-
if (z > 1.0) vm = Math.min(1, (z - 1.0) / 2.0);
|
|
320
|
-
}
|
|
321
|
-
const tm = Math.max(km, vm);
|
|
322
|
-
const ds = baseLevelActivation(engram.accessCount, ageDays);
|
|
323
|
-
const rh = associations.length > 0 ? Math.min(associations.reduce((s, a) => s + a.weight, 0) / associations.length, 0.5) : 0;
|
|
324
|
-
const tn = Math.min(softplus(ds + rh), 3.0) / 3.0;
|
|
325
|
-
const rg = tm > 0.1 ? tm : 0.0;
|
|
326
|
-
const comp = (0.6 * tm + 0.4 * tn * rg) * engram.confidence;
|
|
327
|
-
scored.push({ engram, score: comp, phaseScores: { textMatch: tm, vectorMatch: vm, decayScore: ds, hebbianBoost: rh, graphBoost: 0, confidenceGate: engram.confidence, composite: comp, rerankerScore: 0 }, associations });
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// Phase 3.7: Entity-Bridge boost — boost scored candidates that share entity tags
|
|
334
|
-
// with the most query-relevant result. Only bridge from the single best text-match
|
|
335
|
-
// to avoid pulling in unrelated entities from tangentially-matching results.
|
|
336
|
-
{
|
|
337
|
-
// Find the result with the highest textMatch (most query-relevant, not just highest score)
|
|
338
|
-
// Gate: only bridge when anchor has meaningful text relevance (> 0.15)
|
|
339
|
-
// Adaptive: scale bridge boost inversely with candidate pool size to prevent
|
|
340
|
-
// over-boosting in large memory pools where many items share entity tags
|
|
341
|
-
const sortedByTextMatch = scored
|
|
342
|
-
.filter(r => r.phaseScores.textMatch > 0.15)
|
|
343
|
-
.sort((a, b) => b.phaseScores.textMatch - a.phaseScores.textMatch);
|
|
344
|
-
|
|
345
|
-
// Bridge from top 2 text-matched results (IDF handles weighting)
|
|
346
|
-
const bridgeAnchors = sortedByTextMatch.slice(0, 2);
|
|
347
|
-
|
|
348
|
-
if (bridgeAnchors.length > 0) {
|
|
349
|
-
const entityTags = new Set<string>();
|
|
350
|
-
const anchorIds = new Set(bridgeAnchors.map(r => r.engram.id));
|
|
351
|
-
|
|
352
|
-
for (const item of bridgeAnchors) {
|
|
353
|
-
for (const tag of item.engram.tags) {
|
|
354
|
-
const t = tag.toLowerCase();
|
|
355
|
-
// Skip non-entity tags: turn IDs, session tags, dialogue IDs, generic speaker labels
|
|
356
|
-
if (/^t\d+$/.test(t) || t.startsWith('session-') || t.startsWith('dia_') || t.length < 3) continue;
|
|
357
|
-
if (/^speaker\d*$/.test(t)) continue; // Generic speaker labels are too broad
|
|
358
|
-
entityTags.add(t);
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// Document frequency filter: remove tags appearing in >30% of items (too common)
|
|
363
|
-
// This prevents speaker names in 2-person conversations from being used as bridges
|
|
364
|
-
if (entityTags.size > 0 && scored.length > 10) {
|
|
365
|
-
const tagFreqs = new Map<string, number>();
|
|
366
|
-
for (const item of scored) {
|
|
367
|
-
const seen = new Set<string>();
|
|
368
|
-
for (const tag of item.engram.tags) {
|
|
369
|
-
const t = tag.toLowerCase();
|
|
370
|
-
if (entityTags.has(t) && !seen.has(t)) {
|
|
371
|
-
seen.add(t);
|
|
372
|
-
tagFreqs.set(t, (tagFreqs.get(t) ?? 0) + 1);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
const maxFreq = scored.length * 0.30;
|
|
377
|
-
for (const [tag, freq] of tagFreqs) {
|
|
378
|
-
if (freq > maxFreq) entityTags.delete(tag);
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
if (entityTags.size > 0) {
|
|
383
|
-
for (const item of scored) {
|
|
384
|
-
if (anchorIds.has(item.engram.id)) continue;
|
|
385
|
-
|
|
386
|
-
const engramTags = new Set(item.engram.tags.map((t: string) => t.toLowerCase()));
|
|
387
|
-
let sharedEntities = 0;
|
|
388
|
-
for (const et of entityTags) {
|
|
389
|
-
if (engramTags.has(et)) sharedEntities++;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
if (sharedEntities > 0) {
|
|
393
|
-
// Flat bridge boost per shared entity
|
|
394
|
-
const bridgeBoost = Math.min(sharedEntities * 0.15, 0.4);
|
|
395
|
-
item.score += bridgeBoost;
|
|
396
|
-
item.phaseScores.composite += bridgeBoost;
|
|
397
|
-
item.phaseScores.graphBoost += bridgeBoost;
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
// Phase 4+5: Graph walk — boost engrams connected to high-scoring ones
|
|
405
|
-
// Only walk from engrams that had text relevance (composite > 0 pre-walk)
|
|
406
|
-
const sorted = scored.sort((a, b) => b.score - a.score);
|
|
407
|
-
const topN = sorted.slice(0, limit * 3);
|
|
408
|
-
this.graphWalk(topN, 2, 0.3);
|
|
409
|
-
|
|
410
|
-
// Phase 6: Initial filter and sort for re-ranking pool
|
|
411
|
-
const pool = topN
|
|
412
|
-
.filter(r => r.score >= minScore)
|
|
413
|
-
.sort((a, b) => b.score - a.score);
|
|
414
|
-
|
|
415
|
-
// Phase 7: Cross-encoder re-ranking — scores (query, passage) pairs directly
|
|
416
|
-
// Widens the pool to find relevant results that keyword matching missed
|
|
417
|
-
const rerankPool = pool.slice(0, Math.max(limit * 3, 30));
|
|
418
|
-
|
|
419
|
-
if (useReranker && rerankPool.length > 0) {
|
|
420
|
-
try {
|
|
421
|
-
const passages = rerankPool.map(r =>
|
|
422
|
-
`${r.engram.concept}: ${r.engram.content}`
|
|
423
|
-
);
|
|
424
|
-
const rerankResults = await rerank(query.context, passages);
|
|
425
|
-
|
|
426
|
-
// Adaptive reranker blend (Codex recommendation):
|
|
427
|
-
// When BM25/text signals are strong, trust them more; when weak, lean on reranker.
|
|
428
|
-
const bm25Max = Math.max(...rerankPool.map(r => r.phaseScores.textMatch));
|
|
429
|
-
const rerankWeight = Math.min(0.7, Math.max(0.3, 0.3 + 0.4 * (1 - bm25Max)));
|
|
430
|
-
const compositeWeight = 1 - rerankWeight;
|
|
431
|
-
|
|
432
|
-
for (const rr of rerankResults) {
|
|
433
|
-
const item = rerankPool[rr.index];
|
|
434
|
-
item.phaseScores.rerankerScore = rr.score;
|
|
435
|
-
item.score = compositeWeight * item.phaseScores.composite + rerankWeight * rr.score;
|
|
436
|
-
}
|
|
437
|
-
} catch {
|
|
438
|
-
// Re-ranker unavailable — keep original scores
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
// Phase 8: Multi-channel OOD detection + agreement gate
|
|
443
|
-
// Requires at least 2 of 3 retrieval channels to agree the query is in-domain.
|
|
444
|
-
if (rerankPool.length >= 3) {
|
|
445
|
-
const topBM25 = Math.max(...rerankPool.map(r => bm25ScoreMap.get(r.engram.id) ?? 0));
|
|
446
|
-
const topVector = queryEmbedding
|
|
447
|
-
? Math.max(...rerankPool.map(r => r.phaseScores.vectorMatch))
|
|
448
|
-
: 0;
|
|
449
|
-
const topReranker = Math.max(...rerankPool.map(r => r.phaseScores.rerankerScore));
|
|
450
|
-
|
|
451
|
-
const bm25Ok = topBM25 > 0.3;
|
|
452
|
-
const vectorOk = topVector > 0.05;
|
|
453
|
-
const rerankerOk = topReranker > 0.25;
|
|
454
|
-
const channelsAgreeing = (bm25Ok ? 1 : 0) + (vectorOk ? 1 : 0) + (rerankerOk ? 1 : 0);
|
|
455
|
-
|
|
456
|
-
const rerankerScores = rerankPool
|
|
457
|
-
.map(r => r.phaseScores.rerankerScore)
|
|
458
|
-
.sort((a, b) => b - a);
|
|
459
|
-
const margin = rerankerScores.length >= 2
|
|
460
|
-
? rerankerScores[0] - rerankerScores[1]
|
|
461
|
-
: rerankerScores[0];
|
|
462
|
-
|
|
463
|
-
const maxRawCosine = queryEmbedding && simValues.length > 0
|
|
464
|
-
? Math.max(...simValues)
|
|
465
|
-
: 1.0;
|
|
466
|
-
|
|
467
|
-
// Stricter gate when caller explicitly requests abstention (e.g., noise filter queries)
|
|
468
|
-
const requiredChannels = abstentionThreshold > 0 ? 3 : 2;
|
|
469
|
-
|
|
470
|
-
// Hard abstention: fewer than required channels agree AND semantic drift is high
|
|
471
|
-
if (channelsAgreeing < requiredChannels && maxRawCosine < (simMean + simStdDev * 1.5)) {
|
|
472
|
-
return [];
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
// Soft penalty: only 1 channel agrees or margin is thin
|
|
476
|
-
if (channelsAgreeing < 2 || margin < 0.05) {
|
|
477
|
-
// If caller explicitly requested abstention, honor it when agreement is weak
|
|
478
|
-
if (abstentionThreshold > 0) {
|
|
479
|
-
return [];
|
|
480
|
-
}
|
|
481
|
-
for (const item of rerankPool) {
|
|
482
|
-
item.score *= 0.4;
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
// Legacy abstention gate (when explicitly requested)
|
|
488
|
-
if (abstentionThreshold > 0 && rerankPool.length >= 3) {
|
|
489
|
-
const topRerankerScores = rerankPool
|
|
490
|
-
.map(r => r.phaseScores.rerankerScore)
|
|
491
|
-
.sort((a, b) => b - a)
|
|
492
|
-
.slice(0, 5);
|
|
493
|
-
const maxScore = topRerankerScores[0];
|
|
494
|
-
const meanScore = topRerankerScores.reduce((s, v) => s + v, 0) / topRerankerScores.length;
|
|
495
|
-
const variance = topRerankerScores.reduce((s, v) => s + (v - meanScore) ** 2, 0) / topRerankerScores.length;
|
|
496
|
-
|
|
497
|
-
if (maxScore < abstentionThreshold || (maxScore < 0.5 && variance < 0.01)) {
|
|
498
|
-
return [];
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
// Phase 8c: Supersession penalty — superseded memories are deprioritized.
|
|
503
|
-
// They aren't wrong (that's retraction), just outdated.
|
|
504
|
-
for (const item of rerankPool) {
|
|
505
|
-
if (item.engram.supersededBy) {
|
|
506
|
-
item.score *= 0.15; // Severe down-rank — successor should dominate
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
// Phase 9: Final sort, limit, explain
|
|
511
|
-
const results: ActivationResult[] = rerankPool
|
|
512
|
-
.sort((a, b) => b.score - a.score)
|
|
513
|
-
.slice(0, limit)
|
|
514
|
-
.map(r => ({
|
|
515
|
-
engram: r.engram,
|
|
516
|
-
score: r.score,
|
|
517
|
-
phaseScores: r.phaseScores,
|
|
518
|
-
why: this.explain(r.phaseScores, r.engram, r.associations),
|
|
519
|
-
associations: r.associations,
|
|
520
|
-
}));
|
|
521
|
-
|
|
522
|
-
const activatedIds = results.map(r => r.engram.id);
|
|
523
|
-
|
|
524
|
-
// Side effects: touch, co-activate, Hebbian update (skip for internal/system calls)
|
|
525
|
-
if (!query.internal) {
|
|
526
|
-
for (const id of activatedIds) {
|
|
527
|
-
this.store.touchEngram(id);
|
|
528
|
-
}
|
|
529
|
-
this.coActivationBuffer.pushBatch(activatedIds);
|
|
530
|
-
this.updateHebbianWeights();
|
|
531
|
-
|
|
532
|
-
// Log activation event for eval
|
|
533
|
-
const latencyMs = performance.now() - startTime;
|
|
534
|
-
this.store.logActivationEvent({
|
|
535
|
-
id: randomUUID(),
|
|
536
|
-
agentId: query.agentId,
|
|
537
|
-
timestamp: new Date(),
|
|
538
|
-
context: query.context,
|
|
539
|
-
resultsReturned: results.length,
|
|
540
|
-
topScore: results.length > 0 ? results[0].score : 0,
|
|
541
|
-
latencyMs,
|
|
542
|
-
engramIds: activatedIds,
|
|
543
|
-
});
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
return results;
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
/**
|
|
550
|
-
* Beam search graph walk — replaces naive BFS.
|
|
551
|
-
* Scores paths (not just nodes), uses query-dependent edge filtering,
|
|
552
|
-
* and supports deeper exploration with focused beams.
|
|
553
|
-
*/
|
|
554
|
-
private graphWalk(
|
|
555
|
-
scored: { engram: Engram; score: number; phaseScores: PhaseScores; associations: Association[] }[],
|
|
556
|
-
maxDepth: number,
|
|
557
|
-
hopPenalty: number
|
|
558
|
-
): void {
|
|
559
|
-
const scoreMap = new Map(scored.map(s => [s.engram.id, s]));
|
|
560
|
-
const MAX_TOTAL_BOOST = 0.25; // Slightly higher cap for beam search (deeper paths earn it)
|
|
561
|
-
const BEAM_WIDTH = 15;
|
|
562
|
-
|
|
563
|
-
// Seed the beam with high-scoring, text-relevant items
|
|
564
|
-
const beam = scored
|
|
565
|
-
.filter(item => item.phaseScores.textMatch >= 0.15)
|
|
566
|
-
.sort((a, b) => b.score - a.score)
|
|
567
|
-
.slice(0, BEAM_WIDTH);
|
|
568
|
-
|
|
569
|
-
// Track which engrams have been explored (avoid cycles)
|
|
570
|
-
const explored = new Set<string>();
|
|
571
|
-
|
|
572
|
-
for (let depth = 0; depth < maxDepth; depth++) {
|
|
573
|
-
const nextBeam: typeof beam = [];
|
|
574
|
-
|
|
575
|
-
for (const item of beam) {
|
|
576
|
-
if (explored.has(item.engram.id)) continue;
|
|
577
|
-
explored.add(item.engram.id);
|
|
578
|
-
|
|
579
|
-
// Get associations — for depth > 0, fetch from store if not in scored set
|
|
580
|
-
const associations = item.associations.length > 0
|
|
581
|
-
? item.associations
|
|
582
|
-
: this.store.getAssociationsFor(item.engram.id);
|
|
583
|
-
|
|
584
|
-
for (const assoc of associations) {
|
|
585
|
-
const neighborId = assoc.fromEngramId === item.engram.id
|
|
586
|
-
? assoc.toEngramId
|
|
587
|
-
: assoc.fromEngramId;
|
|
588
|
-
|
|
589
|
-
if (explored.has(neighborId)) continue;
|
|
590
|
-
|
|
591
|
-
const neighbor = scoreMap.get(neighborId);
|
|
592
|
-
if (!neighbor) continue;
|
|
593
|
-
|
|
594
|
-
// Query-dependent edge filtering: neighbor must have SOME relevance
|
|
595
|
-
// (textMatch > 0.05 for deeper hops, relaxed from 0.1)
|
|
596
|
-
const relevanceFloor = depth === 0 ? 0.1 : 0.05;
|
|
597
|
-
if (neighbor.phaseScores.textMatch < relevanceFloor) continue;
|
|
598
|
-
|
|
599
|
-
// Skip if neighbor already at boost cap
|
|
600
|
-
if (neighbor.phaseScores.graphBoost >= MAX_TOTAL_BOOST) continue;
|
|
601
|
-
|
|
602
|
-
// Path score: source score * edge weight * hop penalty^(depth+1)
|
|
603
|
-
const normalizedWeight = Math.min(assoc.weight, 5.0) / 5.0;
|
|
604
|
-
const pathScore = item.score * normalizedWeight * Math.pow(hopPenalty, depth + 1);
|
|
605
|
-
|
|
606
|
-
const boost = Math.min(pathScore, 0.15, MAX_TOTAL_BOOST - neighbor.phaseScores.graphBoost);
|
|
607
|
-
if (boost > 0.001) {
|
|
608
|
-
neighbor.score += boost;
|
|
609
|
-
neighbor.phaseScores.graphBoost += boost;
|
|
610
|
-
nextBeam.push(neighbor);
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
// Prune beam for next depth level
|
|
616
|
-
if (nextBeam.length === 0) break;
|
|
617
|
-
beam.length = 0;
|
|
618
|
-
beam.push(...nextBeam
|
|
619
|
-
.sort((a, b) => b.score - a.score)
|
|
620
|
-
.slice(0, BEAM_WIDTH)
|
|
621
|
-
);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
private updateHebbianWeights(): void {
|
|
626
|
-
const pairs = this.coActivationBuffer.getCoActivatedPairs(10_000);
|
|
627
|
-
// Deduplicate pairs to prevent repeated strengthening
|
|
628
|
-
const seen = new Set<string>();
|
|
629
|
-
for (const [a, b] of pairs) {
|
|
630
|
-
const key = a < b ? `${a}:${b}` : `${b}:${a}`;
|
|
631
|
-
if (seen.has(key)) continue;
|
|
632
|
-
seen.add(key);
|
|
633
|
-
|
|
634
|
-
const existing = this.store.getAssociation(a, b) ?? this.store.getAssociation(b, a);
|
|
635
|
-
const currentWeight = existing?.weight ?? 0.1;
|
|
636
|
-
const newWeight = strengthenAssociation(currentWeight);
|
|
637
|
-
this.store.upsertAssociation(a, b, newWeight, 'hebbian');
|
|
638
|
-
this.store.upsertAssociation(b, a, newWeight, 'hebbian');
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
private explain(phases: PhaseScores, engram: Engram, associations: Association[]): string {
|
|
643
|
-
const parts: string[] = [];
|
|
644
|
-
parts.push(`composite=${phases.composite.toFixed(3)}`);
|
|
645
|
-
if (phases.textMatch > 0) parts.push(`text=${phases.textMatch.toFixed(2)}`);
|
|
646
|
-
if (phases.vectorMatch > 0) parts.push(`vector=${phases.vectorMatch.toFixed(2)}`);
|
|
647
|
-
parts.push(`decay=${phases.decayScore.toFixed(2)}`);
|
|
648
|
-
if (phases.hebbianBoost > 0) parts.push(`hebbian=${phases.hebbianBoost.toFixed(2)}`);
|
|
649
|
-
if (phases.graphBoost > 0) parts.push(`graph=${phases.graphBoost.toFixed(2)}`);
|
|
650
|
-
if (phases.rerankerScore > 0) parts.push(`reranker=${phases.rerankerScore.toFixed(2)}`);
|
|
651
|
-
parts.push(`conf=${phases.confidenceGate.toFixed(2)}`);
|
|
652
|
-
parts.push(`access=${engram.accessCount}`);
|
|
653
|
-
if (associations.length > 0) parts.push(`edges=${associations.length}`);
|
|
654
|
-
return parts.join(' | ');
|
|
655
|
-
}
|
|
656
|
-
}
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Activation Pipeline — the core retrieval engine.
|
|
5
|
+
*
|
|
6
|
+
* 10-phase cognitive retrieval pipeline:
|
|
7
|
+
* 0. Query expansion (flan-t5-small synonym generation)
|
|
8
|
+
* 1. Vector embedding (MiniLM 384d)
|
|
9
|
+
* 2. Parallel retrieval (FTS5/BM25 + vector pool)
|
|
10
|
+
* 3. Scoring (BM25, Jaccard, z-score vector, entity-bridge boost)
|
|
11
|
+
* 4. Rocchio pseudo-relevance feedback (expand + re-search BM25)
|
|
12
|
+
* 5. ACT-R temporal decay
|
|
13
|
+
* 6. Hebbian boost (co-activation strength)
|
|
14
|
+
* 7. Composite scoring with confidence gating
|
|
15
|
+
* 8. Beam search graph walk (depth 2, hop penalty)
|
|
16
|
+
* 9. Cross-encoder reranking (ms-marco-MiniLM, adaptive blend)
|
|
17
|
+
* 10. Abstention gate
|
|
18
|
+
*
|
|
19
|
+
* Logs activation events for eval metrics.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { randomUUID } from 'node:crypto';
|
|
23
|
+
import { baseLevelActivation, softplus } from '../core/decay.js';
|
|
24
|
+
import { strengthenAssociation, CoActivationBuffer } from '../core/hebbian.js';
|
|
25
|
+
import { embed, cosineSimilarity } from '../core/embeddings.js';
|
|
26
|
+
import { rerank } from '../core/reranker.js';
|
|
27
|
+
import { expandQuery } from '../core/query-expander.js';
|
|
28
|
+
import type {
|
|
29
|
+
Engram, ActivationResult, ActivationQuery, Association, PhaseScores,
|
|
30
|
+
} from '../types/index.js';
|
|
31
|
+
import type { EngramStore } from '../storage/sqlite.js';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Common English stopwords — filtered from similarity calculations.
|
|
35
|
+
* These words carry no semantic signal for memory retrieval.
|
|
36
|
+
*/
|
|
37
|
+
const STOPWORDS = new Set([
|
|
38
|
+
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
|
|
39
|
+
'her', 'was', 'one', 'our', 'out', 'has', 'have', 'been', 'from', 'that',
|
|
40
|
+
'this', 'with', 'they', 'will', 'each', 'make', 'like', 'then', 'than',
|
|
41
|
+
'them', 'some', 'what', 'when', 'where', 'which', 'who', 'how', 'use',
|
|
42
|
+
'into', 'does', 'also', 'just', 'more', 'over', 'such', 'only', 'very',
|
|
43
|
+
'about', 'after', 'being', 'between', 'could', 'during', 'before',
|
|
44
|
+
'should', 'would', 'their', 'there', 'these', 'those', 'through',
|
|
45
|
+
'because', 'using', 'other',
|
|
46
|
+
]);
|
|
47
|
+
|
|
48
|
+
function tokenize(text: string): Set<string> {
|
|
49
|
+
return new Set(
|
|
50
|
+
text.toLowerCase()
|
|
51
|
+
.split(/\s+/)
|
|
52
|
+
.filter(w => w.length > 2 && !STOPWORDS.has(w))
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Jaccard similarity between two word sets: |intersection| / |union|
|
|
58
|
+
*/
|
|
59
|
+
function jaccard(a: Set<string>, b: Set<string>): number {
|
|
60
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
61
|
+
let intersection = 0;
|
|
62
|
+
for (const w of a) {
|
|
63
|
+
if (b.has(w)) intersection++;
|
|
64
|
+
}
|
|
65
|
+
const union = a.size + b.size - intersection;
|
|
66
|
+
return union > 0 ? intersection / union : 0;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export class ActivationEngine {
|
|
70
|
+
private store: EngramStore;
|
|
71
|
+
private coActivationBuffer: CoActivationBuffer;
|
|
72
|
+
|
|
73
|
+
constructor(store: EngramStore) {
|
|
74
|
+
this.store = store;
|
|
75
|
+
this.coActivationBuffer = new CoActivationBuffer(50);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Activate — retrieve the most cognitively relevant engrams for a context.
|
|
80
|
+
*/
|
|
81
|
+
async activate(query: ActivationQuery): Promise<ActivationResult[]> {
|
|
82
|
+
const startTime = performance.now();
|
|
83
|
+
const limit = query.limit ?? 10;
|
|
84
|
+
const minScore = query.minScore ?? 0.01; // Default: filter out zero-relevance results
|
|
85
|
+
const useReranker = query.useReranker ?? true;
|
|
86
|
+
const useExpansion = query.useExpansion ?? true;
|
|
87
|
+
const abstentionThreshold = query.abstentionThreshold ?? 0;
|
|
88
|
+
|
|
89
|
+
// Phase -1: Coref expansion — if query has pronouns, append recent entity names
|
|
90
|
+
// Helps conversational recall where "she/he/they/it" refers to a named entity.
|
|
91
|
+
let queryContext = query.context;
|
|
92
|
+
const pronounPattern = /\b(she|he|they|her|his|him|their|it|that|this|there)\b/i;
|
|
93
|
+
if (pronounPattern.test(queryContext)) {
|
|
94
|
+
// Pull recent entity names from the agent's most-accessed memories
|
|
95
|
+
try {
|
|
96
|
+
const recentEntities = this.store.getEngramsByAgent(query.agentId, 'active')
|
|
97
|
+
.sort((a, b) => b.accessCount - a.accessCount)
|
|
98
|
+
.slice(0, 10)
|
|
99
|
+
.flatMap(e => e.tags.filter(t => t.length >= 3 && !/^(session-|low-|D\d)/.test(t)))
|
|
100
|
+
.filter((v, i, a) => a.indexOf(v) === i)
|
|
101
|
+
.slice(0, 5);
|
|
102
|
+
if (recentEntities.length > 0) {
|
|
103
|
+
queryContext = `${queryContext} ${recentEntities.join(' ')}`;
|
|
104
|
+
}
|
|
105
|
+
} catch { /* non-fatal */ }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Phase 0: Query expansion — add related terms to improve BM25 recall
|
|
109
|
+
let searchContext = queryContext;
|
|
110
|
+
if (useExpansion) {
|
|
111
|
+
try {
|
|
112
|
+
searchContext = await expandQuery(query.context);
|
|
113
|
+
} catch {
|
|
114
|
+
// Expansion unavailable — use original query
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Phase 1: Embed query for vector similarity (uses coref-expanded context)
|
|
119
|
+
let queryEmbedding: number[] | null = null;
|
|
120
|
+
try {
|
|
121
|
+
queryEmbedding = await embed(queryContext);
|
|
122
|
+
} catch {
|
|
123
|
+
// Embedding unavailable — fall back to text-only matching
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Phase 2: Parallel retrieval — dual BM25 + all active engrams
|
|
127
|
+
// Two-pass BM25: (1) keyword-stripped query for precision, (2) expanded query for recall.
|
|
128
|
+
const keywordQuery = Array.from(tokenize(query.context)).join(' ');
|
|
129
|
+
const bm25Keyword = keywordQuery.length > 2
|
|
130
|
+
? this.store.searchBM25WithRank(query.agentId, keywordQuery, limit * 3)
|
|
131
|
+
: [];
|
|
132
|
+
const bm25Expanded = this.store.searchBM25WithRank(query.agentId, searchContext, limit * 3);
|
|
133
|
+
|
|
134
|
+
// Merge: take the best BM25 score per engram from either pass
|
|
135
|
+
const bm25ScoreMap = new Map<string, number>();
|
|
136
|
+
const bm25EngramMap = new Map<string, any>();
|
|
137
|
+
for (const r of [...bm25Keyword, ...bm25Expanded]) {
|
|
138
|
+
const existing = bm25ScoreMap.get(r.engram.id) ?? 0;
|
|
139
|
+
if (r.bm25Score > existing) {
|
|
140
|
+
bm25ScoreMap.set(r.engram.id, r.bm25Score);
|
|
141
|
+
bm25EngramMap.set(r.engram.id, r.engram);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const bm25Ranked = Array.from(bm25EngramMap.entries()).map(([id, engram]) => ({
|
|
145
|
+
engram, bm25Score: bm25ScoreMap.get(id) ?? 0,
|
|
146
|
+
}));
|
|
147
|
+
|
|
148
|
+
const allActive = this.store.getEngramsByAgent(
|
|
149
|
+
query.agentId,
|
|
150
|
+
query.includeStaging ? undefined : 'active',
|
|
151
|
+
query.includeRetracted ?? false
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
// Merge candidates (deduplicate)
|
|
155
|
+
const candidateMap = new Map<string, Engram>();
|
|
156
|
+
for (const r of bm25Ranked) candidateMap.set(r.engram.id, r.engram);
|
|
157
|
+
for (const e of allActive) candidateMap.set(e.id, e);
|
|
158
|
+
const candidates = Array.from(candidateMap.values());
|
|
159
|
+
|
|
160
|
+
if (candidates.length === 0) return [];
|
|
161
|
+
|
|
162
|
+
// Tokenize query once
|
|
163
|
+
const queryTokens = tokenize(query.context);
|
|
164
|
+
|
|
165
|
+
// Phase 3a: Compute raw cosine similarities for adaptive normalization
|
|
166
|
+
const rawCosineSims = new Map<string, number>();
|
|
167
|
+
if (queryEmbedding) {
|
|
168
|
+
for (const engram of candidates) {
|
|
169
|
+
if (engram.embedding) {
|
|
170
|
+
rawCosineSims.set(engram.id, cosineSimilarity(queryEmbedding, engram.embedding));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Compute distribution stats for model-agnostic normalization
|
|
176
|
+
const simValues = Array.from(rawCosineSims.values());
|
|
177
|
+
const simMean = simValues.length > 0
|
|
178
|
+
? simValues.reduce((a, b) => a + b, 0) / simValues.length : 0;
|
|
179
|
+
const rawStdDev = simValues.length > 1
|
|
180
|
+
? Math.sqrt(simValues.reduce((sum, s) => sum + (s - simMean) ** 2, 0) / simValues.length) : 0.15;
|
|
181
|
+
// Floor stddev at 0.10 to prevent z-score inflation with small candidate pools
|
|
182
|
+
const simStdDev = Math.max(rawStdDev, 0.10);
|
|
183
|
+
|
|
184
|
+
// Phase 3b: Score each candidate with per-phase breakdown
|
|
185
|
+
const scored = candidates.map(engram => {
|
|
186
|
+
const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
|
|
187
|
+
const associations = this.store.getAssociationsFor(engram.id);
|
|
188
|
+
|
|
189
|
+
// --- Text relevance (keyword signals) ---
|
|
190
|
+
|
|
191
|
+
// Signal 1: BM25 continuous score (0-1, from FTS5 rank)
|
|
192
|
+
const bm25Score = bm25ScoreMap.get(engram.id) ?? 0;
|
|
193
|
+
|
|
194
|
+
// Signal 2: Jaccard similarity with stopword filtering
|
|
195
|
+
const conceptTokens = tokenize(engram.concept);
|
|
196
|
+
const contentTokens = tokenize(engram.content);
|
|
197
|
+
const conceptJaccard = jaccard(queryTokens, conceptTokens);
|
|
198
|
+
const contentJaccard = jaccard(queryTokens, contentTokens);
|
|
199
|
+
const jaccardScore = 0.6 * conceptJaccard + 0.4 * contentJaccard;
|
|
200
|
+
|
|
201
|
+
// Signal 3: Concept exact match bonus (up to 0.3)
|
|
202
|
+
const conceptOverlap = conceptTokens.size > 0
|
|
203
|
+
? [...conceptTokens].filter(w => queryTokens.has(w)).length / conceptTokens.size
|
|
204
|
+
: 0;
|
|
205
|
+
const conceptBonus = conceptOverlap * 0.3;
|
|
206
|
+
|
|
207
|
+
const keywordMatch = Math.min(Math.max(bm25Score, jaccardScore) + conceptBonus, 1.0);
|
|
208
|
+
|
|
209
|
+
// --- Vector similarity (semantic signal) ---
|
|
210
|
+
// Two-stage: absolute floor prevents noise, then z-score ranks within matches.
|
|
211
|
+
// Stage 1: Raw cosine must exceed mean + 1 stddev (absolute relevance gate)
|
|
212
|
+
// Stage 2: Z-score maps relative position to 0-1 for ranking quality
|
|
213
|
+
let vectorMatch = 0;
|
|
214
|
+
const rawSim = rawCosineSims.get(engram.id);
|
|
215
|
+
if (rawSim !== undefined) {
|
|
216
|
+
const zScore = (rawSim - simMean) / simStdDev;
|
|
217
|
+
// Gate: must be at least 1 stddev above mean to be considered a match
|
|
218
|
+
if (zScore > 1.0) {
|
|
219
|
+
// Map z=1..3 → 0..1 linearly
|
|
220
|
+
vectorMatch = Math.min(1, (zScore - 1.0) / 2.0);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Combined text match: best of keyword and vector signals
|
|
225
|
+
const textMatch = Math.max(keywordMatch, vectorMatch);
|
|
226
|
+
|
|
227
|
+
// --- Temporal signals ---
|
|
228
|
+
|
|
229
|
+
// ACT-R decay — confidence + replay modulated (synaptic tagging)
|
|
230
|
+
// High-confidence memories decay slower. Heavily-accessed memories also resist decay.
|
|
231
|
+
// Default exponent: 0.5. High confidence (0.8+): 0.3. High access (10+): further -0.05.
|
|
232
|
+
const confMod = 0.2 * Math.max(0, (engram.confidence - 0.5) / 0.5);
|
|
233
|
+
const replayMod = Math.min(0.1, 0.05 * Math.log1p(engram.accessCount));
|
|
234
|
+
const decayExponent = Math.max(0.2, 0.5 - confMod - replayMod);
|
|
235
|
+
const decayScore = baseLevelActivation(engram.accessCount, ageDays, decayExponent);
|
|
236
|
+
|
|
237
|
+
// Hebbian boost from associations — capped to prevent popular memories
|
|
238
|
+
// from dominating regardless of query relevance
|
|
239
|
+
const rawHebbian = associations.length > 0
|
|
240
|
+
? associations.reduce((sum, a) => sum + a.weight, 0) / associations.length
|
|
241
|
+
: 0;
|
|
242
|
+
const hebbianBoost = Math.min(rawHebbian, 0.5);
|
|
243
|
+
|
|
244
|
+
// Centrality signal — well-connected memories (high weighted degree)
|
|
245
|
+
// get a small boost. This makes consolidation edges matter for retrieval.
|
|
246
|
+
// Log-scaled to prevent hub domination: 10 edges ≈ 0.05 boost, 50 ≈ 0.08
|
|
247
|
+
const weightedDegree = associations.reduce((sum, a) => sum + a.weight, 0);
|
|
248
|
+
const centralityBoost = associations.length > 0
|
|
249
|
+
? Math.min(0.1, 0.03 * Math.log1p(weightedDegree))
|
|
250
|
+
: 0;
|
|
251
|
+
|
|
252
|
+
// Confidence gate — multiplicative quality signal
|
|
253
|
+
const confidenceGate = engram.confidence;
|
|
254
|
+
|
|
255
|
+
// Feedback bonus — memories confirmed useful via explicit feedback get a
|
|
256
|
+
// direct additive boost. Models how a senior dev "just knows" certain things
|
|
257
|
+
// are important. Confidence > 0.6 means at least 2+ positive feedbacks.
|
|
258
|
+
// Scales: conf 0.6→0.03, 0.7→0.06, 0.8→0.09, 1.0→0.15
|
|
259
|
+
const feedbackBonus = engram.confidence > 0.55
|
|
260
|
+
? Math.min(0.15, 0.3 * Math.max(0, engram.confidence - 0.5))
|
|
261
|
+
: 0;
|
|
262
|
+
|
|
263
|
+
// --- Composite score: relevance-gated additive ---
|
|
264
|
+
// Text relevance must be present for temporal signals to contribute.
|
|
265
|
+
// Without text relevance, a memory shouldn't activate regardless of recency.
|
|
266
|
+
// Temporal contribution scales with text relevance (weak match = weak temporal boost).
|
|
267
|
+
const temporalNorm = Math.min(softplus(decayScore + hebbianBoost), 3.0) / 3.0;
|
|
268
|
+
const relevanceGate = textMatch > 0.1 ? textMatch : 0.0; // Proportional gate
|
|
269
|
+
const composite = (0.6 * textMatch + 0.4 * temporalNorm * relevanceGate + centralityBoost * relevanceGate + feedbackBonus * relevanceGate) * confidenceGate;
|
|
270
|
+
|
|
271
|
+
const phaseScores: PhaseScores = {
|
|
272
|
+
textMatch,
|
|
273
|
+
vectorMatch,
|
|
274
|
+
decayScore,
|
|
275
|
+
hebbianBoost,
|
|
276
|
+
graphBoost: 0, // Filled in phase 5
|
|
277
|
+
confidenceGate,
|
|
278
|
+
composite,
|
|
279
|
+
rerankerScore: 0, // Filled in phase 7
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
return { engram, score: composite, phaseScores, associations };
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
// Phase 3.5: Rocchio pseudo-relevance feedback — expand query with top result terms
|
|
286
|
+
// then re-search BM25 to find candidates that keyword search missed
|
|
287
|
+
const preSorted = scored.sort((a, b) => b.score - a.score);
|
|
288
|
+
const topForFeedback = preSorted.slice(0, 3).filter(r => r.phaseScores.textMatch > 0.1);
|
|
289
|
+
if (topForFeedback.length > 0) {
|
|
290
|
+
const feedbackTerms = new Set<string>();
|
|
291
|
+
for (const item of topForFeedback) {
|
|
292
|
+
const tokens = tokenize(item.engram.content);
|
|
293
|
+
for (const t of tokens) {
|
|
294
|
+
if (!queryTokens.has(t) && t.length >= 4) feedbackTerms.add(t);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
// Take top 5 feedback terms and re-search
|
|
298
|
+
const extraTerms = Array.from(feedbackTerms).slice(0, 5).join(' ');
|
|
299
|
+
if (extraTerms) {
|
|
300
|
+
const feedbackBM25 = this.store.searchBM25WithRank(query.agentId, `${searchContext} ${extraTerms}`, limit * 2);
|
|
301
|
+
for (const r of feedbackBM25) {
|
|
302
|
+
if (!candidateMap.has(r.engram.id)) {
|
|
303
|
+
candidateMap.set(r.engram.id, r.engram);
|
|
304
|
+
// Score the new candidate
|
|
305
|
+
const engram = r.engram;
|
|
306
|
+
const ageDays = (Date.now() - engram.createdAt.getTime()) / (1000 * 60 * 60 * 24);
|
|
307
|
+
const associations = this.store.getAssociationsFor(engram.id);
|
|
308
|
+
const cTokens = tokenize(engram.concept);
|
|
309
|
+
const ctTokens = tokenize(engram.content);
|
|
310
|
+
const cJac = jaccard(queryTokens, cTokens);
|
|
311
|
+
const ctJac = jaccard(queryTokens, ctTokens);
|
|
312
|
+
const jSc = 0.6 * cJac + 0.4 * ctJac;
|
|
313
|
+
const cOvlp = cTokens.size > 0 ? [...cTokens].filter(w => queryTokens.has(w)).length / cTokens.size : 0;
|
|
314
|
+
const km = Math.min(Math.max(r.bm25Score, jSc) + cOvlp * 0.3, 1.0);
|
|
315
|
+
let vm = 0;
|
|
316
|
+
const rs = rawCosineSims.get(engram.id) ?? (queryEmbedding && engram.embedding ? cosineSimilarity(queryEmbedding, engram.embedding) : 0);
|
|
317
|
+
if (rs) {
|
|
318
|
+
const z = (rs - simMean) / simStdDev;
|
|
319
|
+
if (z > 1.0) vm = Math.min(1, (z - 1.0) / 2.0);
|
|
320
|
+
}
|
|
321
|
+
const tm = Math.max(km, vm);
|
|
322
|
+
const ds = baseLevelActivation(engram.accessCount, ageDays);
|
|
323
|
+
const rh = associations.length > 0 ? Math.min(associations.reduce((s, a) => s + a.weight, 0) / associations.length, 0.5) : 0;
|
|
324
|
+
const tn = Math.min(softplus(ds + rh), 3.0) / 3.0;
|
|
325
|
+
const rg = tm > 0.1 ? tm : 0.0;
|
|
326
|
+
const comp = (0.6 * tm + 0.4 * tn * rg) * engram.confidence;
|
|
327
|
+
scored.push({ engram, score: comp, phaseScores: { textMatch: tm, vectorMatch: vm, decayScore: ds, hebbianBoost: rh, graphBoost: 0, confidenceGate: engram.confidence, composite: comp, rerankerScore: 0 }, associations });
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// Phase 3.7: Entity-Bridge boost — boost scored candidates that share entity tags
|
|
334
|
+
// with the most query-relevant result. Only bridge from the single best text-match
|
|
335
|
+
// to avoid pulling in unrelated entities from tangentially-matching results.
|
|
336
|
+
{
|
|
337
|
+
// Find the result with the highest textMatch (most query-relevant, not just highest score)
|
|
338
|
+
// Gate: only bridge when anchor has meaningful text relevance (> 0.15)
|
|
339
|
+
// Adaptive: scale bridge boost inversely with candidate pool size to prevent
|
|
340
|
+
// over-boosting in large memory pools where many items share entity tags
|
|
341
|
+
const sortedByTextMatch = scored
|
|
342
|
+
.filter(r => r.phaseScores.textMatch > 0.15)
|
|
343
|
+
.sort((a, b) => b.phaseScores.textMatch - a.phaseScores.textMatch);
|
|
344
|
+
|
|
345
|
+
// Bridge from top 2 text-matched results (IDF handles weighting)
|
|
346
|
+
const bridgeAnchors = sortedByTextMatch.slice(0, 2);
|
|
347
|
+
|
|
348
|
+
if (bridgeAnchors.length > 0) {
|
|
349
|
+
const entityTags = new Set<string>();
|
|
350
|
+
const anchorIds = new Set(bridgeAnchors.map(r => r.engram.id));
|
|
351
|
+
|
|
352
|
+
for (const item of bridgeAnchors) {
|
|
353
|
+
for (const tag of item.engram.tags) {
|
|
354
|
+
const t = tag.toLowerCase();
|
|
355
|
+
// Skip non-entity tags: turn IDs, session tags, dialogue IDs, generic speaker labels
|
|
356
|
+
if (/^t\d+$/.test(t) || t.startsWith('session-') || t.startsWith('dia_') || t.length < 3) continue;
|
|
357
|
+
if (/^speaker\d*$/.test(t)) continue; // Generic speaker labels are too broad
|
|
358
|
+
entityTags.add(t);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Document frequency filter: remove tags appearing in >30% of items (too common)
|
|
363
|
+
// This prevents speaker names in 2-person conversations from being used as bridges
|
|
364
|
+
if (entityTags.size > 0 && scored.length > 10) {
|
|
365
|
+
const tagFreqs = new Map<string, number>();
|
|
366
|
+
for (const item of scored) {
|
|
367
|
+
const seen = new Set<string>();
|
|
368
|
+
for (const tag of item.engram.tags) {
|
|
369
|
+
const t = tag.toLowerCase();
|
|
370
|
+
if (entityTags.has(t) && !seen.has(t)) {
|
|
371
|
+
seen.add(t);
|
|
372
|
+
tagFreqs.set(t, (tagFreqs.get(t) ?? 0) + 1);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
const maxFreq = scored.length * 0.30;
|
|
377
|
+
for (const [tag, freq] of tagFreqs) {
|
|
378
|
+
if (freq > maxFreq) entityTags.delete(tag);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
if (entityTags.size > 0) {
|
|
383
|
+
for (const item of scored) {
|
|
384
|
+
if (anchorIds.has(item.engram.id)) continue;
|
|
385
|
+
|
|
386
|
+
const engramTags = new Set(item.engram.tags.map((t: string) => t.toLowerCase()));
|
|
387
|
+
let sharedEntities = 0;
|
|
388
|
+
for (const et of entityTags) {
|
|
389
|
+
if (engramTags.has(et)) sharedEntities++;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
if (sharedEntities > 0) {
|
|
393
|
+
// Flat bridge boost per shared entity
|
|
394
|
+
const bridgeBoost = Math.min(sharedEntities * 0.15, 0.4);
|
|
395
|
+
item.score += bridgeBoost;
|
|
396
|
+
item.phaseScores.composite += bridgeBoost;
|
|
397
|
+
item.phaseScores.graphBoost += bridgeBoost;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// Phase 4+5: Graph walk — boost engrams connected to high-scoring ones
|
|
405
|
+
// Only walk from engrams that had text relevance (composite > 0 pre-walk)
|
|
406
|
+
const sorted = scored.sort((a, b) => b.score - a.score);
|
|
407
|
+
const topN = sorted.slice(0, limit * 3);
|
|
408
|
+
this.graphWalk(topN, 2, 0.3);
|
|
409
|
+
|
|
410
|
+
// Phase 6: Initial filter and sort for re-ranking pool
|
|
411
|
+
const pool = topN
|
|
412
|
+
.filter(r => r.score >= minScore)
|
|
413
|
+
.sort((a, b) => b.score - a.score);
|
|
414
|
+
|
|
415
|
+
// Phase 7: Cross-encoder re-ranking — scores (query, passage) pairs directly
|
|
416
|
+
// Widens the pool to find relevant results that keyword matching missed
|
|
417
|
+
const rerankPool = pool.slice(0, Math.max(limit * 3, 30));
|
|
418
|
+
|
|
419
|
+
if (useReranker && rerankPool.length > 0) {
|
|
420
|
+
try {
|
|
421
|
+
const passages = rerankPool.map(r =>
|
|
422
|
+
`${r.engram.concept}: ${r.engram.content}`
|
|
423
|
+
);
|
|
424
|
+
const rerankResults = await rerank(query.context, passages);
|
|
425
|
+
|
|
426
|
+
// Adaptive reranker blend (Codex recommendation):
|
|
427
|
+
// When BM25/text signals are strong, trust them more; when weak, lean on reranker.
|
|
428
|
+
const bm25Max = Math.max(...rerankPool.map(r => r.phaseScores.textMatch));
|
|
429
|
+
const rerankWeight = Math.min(0.7, Math.max(0.3, 0.3 + 0.4 * (1 - bm25Max)));
|
|
430
|
+
const compositeWeight = 1 - rerankWeight;
|
|
431
|
+
|
|
432
|
+
for (const rr of rerankResults) {
|
|
433
|
+
const item = rerankPool[rr.index];
|
|
434
|
+
item.phaseScores.rerankerScore = rr.score;
|
|
435
|
+
item.score = compositeWeight * item.phaseScores.composite + rerankWeight * rr.score;
|
|
436
|
+
}
|
|
437
|
+
} catch {
|
|
438
|
+
// Re-ranker unavailable — keep original scores
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// Phase 8: Multi-channel OOD detection + agreement gate
|
|
443
|
+
// Requires at least 2 of 3 retrieval channels to agree the query is in-domain.
|
|
444
|
+
if (rerankPool.length >= 3) {
|
|
445
|
+
const topBM25 = Math.max(...rerankPool.map(r => bm25ScoreMap.get(r.engram.id) ?? 0));
|
|
446
|
+
const topVector = queryEmbedding
|
|
447
|
+
? Math.max(...rerankPool.map(r => r.phaseScores.vectorMatch))
|
|
448
|
+
: 0;
|
|
449
|
+
const topReranker = Math.max(...rerankPool.map(r => r.phaseScores.rerankerScore));
|
|
450
|
+
|
|
451
|
+
const bm25Ok = topBM25 > 0.3;
|
|
452
|
+
const vectorOk = topVector > 0.05;
|
|
453
|
+
const rerankerOk = topReranker > 0.25;
|
|
454
|
+
const channelsAgreeing = (bm25Ok ? 1 : 0) + (vectorOk ? 1 : 0) + (rerankerOk ? 1 : 0);
|
|
455
|
+
|
|
456
|
+
const rerankerScores = rerankPool
|
|
457
|
+
.map(r => r.phaseScores.rerankerScore)
|
|
458
|
+
.sort((a, b) => b - a);
|
|
459
|
+
const margin = rerankerScores.length >= 2
|
|
460
|
+
? rerankerScores[0] - rerankerScores[1]
|
|
461
|
+
: rerankerScores[0];
|
|
462
|
+
|
|
463
|
+
const maxRawCosine = queryEmbedding && simValues.length > 0
|
|
464
|
+
? Math.max(...simValues)
|
|
465
|
+
: 1.0;
|
|
466
|
+
|
|
467
|
+
// Stricter gate when caller explicitly requests abstention (e.g., noise filter queries)
|
|
468
|
+
const requiredChannels = abstentionThreshold > 0 ? 3 : 2;
|
|
469
|
+
|
|
470
|
+
// Hard abstention: fewer than required channels agree AND semantic drift is high
|
|
471
|
+
if (channelsAgreeing < requiredChannels && maxRawCosine < (simMean + simStdDev * 1.5)) {
|
|
472
|
+
return [];
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// Soft penalty: only 1 channel agrees or margin is thin
|
|
476
|
+
if (channelsAgreeing < 2 || margin < 0.05) {
|
|
477
|
+
// If caller explicitly requested abstention, honor it when agreement is weak
|
|
478
|
+
if (abstentionThreshold > 0) {
|
|
479
|
+
return [];
|
|
480
|
+
}
|
|
481
|
+
for (const item of rerankPool) {
|
|
482
|
+
item.score *= 0.4;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// Legacy abstention gate (when explicitly requested)
|
|
488
|
+
if (abstentionThreshold > 0 && rerankPool.length >= 3) {
|
|
489
|
+
const topRerankerScores = rerankPool
|
|
490
|
+
.map(r => r.phaseScores.rerankerScore)
|
|
491
|
+
.sort((a, b) => b - a)
|
|
492
|
+
.slice(0, 5);
|
|
493
|
+
const maxScore = topRerankerScores[0];
|
|
494
|
+
const meanScore = topRerankerScores.reduce((s, v) => s + v, 0) / topRerankerScores.length;
|
|
495
|
+
const variance = topRerankerScores.reduce((s, v) => s + (v - meanScore) ** 2, 0) / topRerankerScores.length;
|
|
496
|
+
|
|
497
|
+
if (maxScore < abstentionThreshold || (maxScore < 0.5 && variance < 0.01)) {
|
|
498
|
+
return [];
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// Phase 8c: Supersession penalty — superseded memories are deprioritized.
|
|
503
|
+
// They aren't wrong (that's retraction), just outdated.
|
|
504
|
+
for (const item of rerankPool) {
|
|
505
|
+
if (item.engram.supersededBy) {
|
|
506
|
+
item.score *= 0.15; // Severe down-rank — successor should dominate
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// Phase 9: Final sort, limit, explain
|
|
511
|
+
const results: ActivationResult[] = rerankPool
|
|
512
|
+
.sort((a, b) => b.score - a.score)
|
|
513
|
+
.slice(0, limit)
|
|
514
|
+
.map(r => ({
|
|
515
|
+
engram: r.engram,
|
|
516
|
+
score: r.score,
|
|
517
|
+
phaseScores: r.phaseScores,
|
|
518
|
+
why: this.explain(r.phaseScores, r.engram, r.associations),
|
|
519
|
+
associations: r.associations,
|
|
520
|
+
}));
|
|
521
|
+
|
|
522
|
+
const activatedIds = results.map(r => r.engram.id);
|
|
523
|
+
|
|
524
|
+
// Side effects: touch, co-activate, Hebbian update (skip for internal/system calls)
|
|
525
|
+
if (!query.internal) {
|
|
526
|
+
for (const id of activatedIds) {
|
|
527
|
+
this.store.touchEngram(id);
|
|
528
|
+
}
|
|
529
|
+
this.coActivationBuffer.pushBatch(activatedIds);
|
|
530
|
+
this.updateHebbianWeights();
|
|
531
|
+
|
|
532
|
+
// Log activation event for eval
|
|
533
|
+
const latencyMs = performance.now() - startTime;
|
|
534
|
+
this.store.logActivationEvent({
|
|
535
|
+
id: randomUUID(),
|
|
536
|
+
agentId: query.agentId,
|
|
537
|
+
timestamp: new Date(),
|
|
538
|
+
context: query.context,
|
|
539
|
+
resultsReturned: results.length,
|
|
540
|
+
topScore: results.length > 0 ? results[0].score : 0,
|
|
541
|
+
latencyMs,
|
|
542
|
+
engramIds: activatedIds,
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
return results;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Beam search graph walk — replaces naive BFS.
|
|
551
|
+
* Scores paths (not just nodes), uses query-dependent edge filtering,
|
|
552
|
+
* and supports deeper exploration with focused beams.
|
|
553
|
+
*/
|
|
554
|
+
private graphWalk(
|
|
555
|
+
scored: { engram: Engram; score: number; phaseScores: PhaseScores; associations: Association[] }[],
|
|
556
|
+
maxDepth: number,
|
|
557
|
+
hopPenalty: number
|
|
558
|
+
): void {
|
|
559
|
+
const scoreMap = new Map(scored.map(s => [s.engram.id, s]));
|
|
560
|
+
const MAX_TOTAL_BOOST = 0.25; // Slightly higher cap for beam search (deeper paths earn it)
|
|
561
|
+
const BEAM_WIDTH = 15;
|
|
562
|
+
|
|
563
|
+
// Seed the beam with high-scoring, text-relevant items
|
|
564
|
+
const beam = scored
|
|
565
|
+
.filter(item => item.phaseScores.textMatch >= 0.15)
|
|
566
|
+
.sort((a, b) => b.score - a.score)
|
|
567
|
+
.slice(0, BEAM_WIDTH);
|
|
568
|
+
|
|
569
|
+
// Track which engrams have been explored (avoid cycles)
|
|
570
|
+
const explored = new Set<string>();
|
|
571
|
+
|
|
572
|
+
for (let depth = 0; depth < maxDepth; depth++) {
|
|
573
|
+
const nextBeam: typeof beam = [];
|
|
574
|
+
|
|
575
|
+
for (const item of beam) {
|
|
576
|
+
if (explored.has(item.engram.id)) continue;
|
|
577
|
+
explored.add(item.engram.id);
|
|
578
|
+
|
|
579
|
+
// Get associations — for depth > 0, fetch from store if not in scored set
|
|
580
|
+
const associations = item.associations.length > 0
|
|
581
|
+
? item.associations
|
|
582
|
+
: this.store.getAssociationsFor(item.engram.id);
|
|
583
|
+
|
|
584
|
+
for (const assoc of associations) {
|
|
585
|
+
const neighborId = assoc.fromEngramId === item.engram.id
|
|
586
|
+
? assoc.toEngramId
|
|
587
|
+
: assoc.fromEngramId;
|
|
588
|
+
|
|
589
|
+
if (explored.has(neighborId)) continue;
|
|
590
|
+
|
|
591
|
+
const neighbor = scoreMap.get(neighborId);
|
|
592
|
+
if (!neighbor) continue;
|
|
593
|
+
|
|
594
|
+
// Query-dependent edge filtering: neighbor must have SOME relevance
|
|
595
|
+
// (textMatch > 0.05 for deeper hops, relaxed from 0.1)
|
|
596
|
+
const relevanceFloor = depth === 0 ? 0.1 : 0.05;
|
|
597
|
+
if (neighbor.phaseScores.textMatch < relevanceFloor) continue;
|
|
598
|
+
|
|
599
|
+
// Skip if neighbor already at boost cap
|
|
600
|
+
if (neighbor.phaseScores.graphBoost >= MAX_TOTAL_BOOST) continue;
|
|
601
|
+
|
|
602
|
+
// Path score: source score * edge weight * hop penalty^(depth+1)
|
|
603
|
+
const normalizedWeight = Math.min(assoc.weight, 5.0) / 5.0;
|
|
604
|
+
const pathScore = item.score * normalizedWeight * Math.pow(hopPenalty, depth + 1);
|
|
605
|
+
|
|
606
|
+
const boost = Math.min(pathScore, 0.15, MAX_TOTAL_BOOST - neighbor.phaseScores.graphBoost);
|
|
607
|
+
if (boost > 0.001) {
|
|
608
|
+
neighbor.score += boost;
|
|
609
|
+
neighbor.phaseScores.graphBoost += boost;
|
|
610
|
+
nextBeam.push(neighbor);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// Prune beam for next depth level
|
|
616
|
+
if (nextBeam.length === 0) break;
|
|
617
|
+
beam.length = 0;
|
|
618
|
+
beam.push(...nextBeam
|
|
619
|
+
.sort((a, b) => b.score - a.score)
|
|
620
|
+
.slice(0, BEAM_WIDTH)
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
private updateHebbianWeights(): void {
|
|
626
|
+
const pairs = this.coActivationBuffer.getCoActivatedPairs(10_000);
|
|
627
|
+
// Deduplicate pairs to prevent repeated strengthening
|
|
628
|
+
const seen = new Set<string>();
|
|
629
|
+
for (const [a, b] of pairs) {
|
|
630
|
+
const key = a < b ? `${a}:${b}` : `${b}:${a}`;
|
|
631
|
+
if (seen.has(key)) continue;
|
|
632
|
+
seen.add(key);
|
|
633
|
+
|
|
634
|
+
const existing = this.store.getAssociation(a, b) ?? this.store.getAssociation(b, a);
|
|
635
|
+
const currentWeight = existing?.weight ?? 0.1;
|
|
636
|
+
const newWeight = strengthenAssociation(currentWeight);
|
|
637
|
+
this.store.upsertAssociation(a, b, newWeight, 'hebbian');
|
|
638
|
+
this.store.upsertAssociation(b, a, newWeight, 'hebbian');
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
private explain(phases: PhaseScores, engram: Engram, associations: Association[]): string {
|
|
643
|
+
const parts: string[] = [];
|
|
644
|
+
parts.push(`composite=${phases.composite.toFixed(3)}`);
|
|
645
|
+
if (phases.textMatch > 0) parts.push(`text=${phases.textMatch.toFixed(2)}`);
|
|
646
|
+
if (phases.vectorMatch > 0) parts.push(`vector=${phases.vectorMatch.toFixed(2)}`);
|
|
647
|
+
parts.push(`decay=${phases.decayScore.toFixed(2)}`);
|
|
648
|
+
if (phases.hebbianBoost > 0) parts.push(`hebbian=${phases.hebbianBoost.toFixed(2)}`);
|
|
649
|
+
if (phases.graphBoost > 0) parts.push(`graph=${phases.graphBoost.toFixed(2)}`);
|
|
650
|
+
if (phases.rerankerScore > 0) parts.push(`reranker=${phases.rerankerScore.toFixed(2)}`);
|
|
651
|
+
parts.push(`conf=${phases.confidenceGate.toFixed(2)}`);
|
|
652
|
+
parts.push(`access=${engram.accessCount}`);
|
|
653
|
+
if (associations.length > 0) parts.push(`edges=${associations.length}`);
|
|
654
|
+
return parts.join(' | ');
|
|
655
|
+
}
|
|
656
|
+
}
|