engram-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +65 -0
- package/Dockerfile +21 -0
- package/EVAL-FRAMEWORK.md +70 -0
- package/EVAL.md +127 -0
- package/LICENSE +17 -0
- package/README.md +309 -0
- package/ROADMAP.md +113 -0
- package/deploy/fly.toml +26 -0
- package/dist/auto-ingest.d.ts +3 -0
- package/dist/auto-ingest.d.ts.map +1 -0
- package/dist/auto-ingest.js +334 -0
- package/dist/auto-ingest.js.map +1 -0
- package/dist/brief.d.ts +45 -0
- package/dist/brief.d.ts.map +1 -0
- package/dist/brief.js +183 -0
- package/dist/brief.js.map +1 -0
- package/dist/claude-watcher.d.ts +3 -0
- package/dist/claude-watcher.d.ts.map +1 -0
- package/dist/claude-watcher.js +385 -0
- package/dist/claude-watcher.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +764 -0
- package/dist/cli.js.map +1 -0
- package/dist/embeddings.d.ts +42 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +145 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/eval.d.ts +2 -0
- package/dist/eval.d.ts.map +1 -0
- package/dist/eval.js +281 -0
- package/dist/eval.js.map +1 -0
- package/dist/extract.d.ts +11 -0
- package/dist/extract.d.ts.map +1 -0
- package/dist/extract.js +139 -0
- package/dist/extract.js.map +1 -0
- package/dist/hosted.d.ts +3 -0
- package/dist/hosted.d.ts.map +1 -0
- package/dist/hosted.js +144 -0
- package/dist/hosted.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest.d.ts +28 -0
- package/dist/ingest.d.ts.map +1 -0
- package/dist/ingest.js +192 -0
- package/dist/ingest.js.map +1 -0
- package/dist/mcp.d.ts +3 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +349 -0
- package/dist/mcp.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +515 -0
- package/dist/server.js.map +1 -0
- package/dist/store.d.ts +87 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +548 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +204 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +77 -0
- package/dist/types.js.map +1 -0
- package/dist/vault.d.ts +116 -0
- package/dist/vault.d.ts.map +1 -0
- package/dist/vault.js +1234 -0
- package/dist/vault.js.map +1 -0
- package/package.json +61 -0
package/dist/vault.js
ADDED
|
@@ -0,0 +1,1234 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { MemoryStore } from './store.js';
|
|
3
|
+
import { RememberInputSchema, RecallInputSchema } from './types.js';
|
|
4
|
+
import { extract } from './extract.js';
|
|
5
|
+
// ============================================================
|
|
6
|
+
// Vault — The public API for Engram
|
|
7
|
+
// ============================================================
|
|
8
|
+
export class Vault {
|
|
9
|
+
store;
|
|
10
|
+
config;
|
|
11
|
+
embedder = null;
|
|
12
|
+
/** Track all in-flight embedding computations so close() can await them */
|
|
13
|
+
pendingEmbeddings = new Set();
|
|
14
|
+
constructor(config, embedder) {
|
|
15
|
+
this.config = config;
|
|
16
|
+
this.embedder = embedder ?? null;
|
|
17
|
+
const dbPath = config.dbPath ?? path.resolve(`engram-${config.owner}.db`);
|
|
18
|
+
this.store = new MemoryStore(dbPath, embedder?.dimensions());
|
|
19
|
+
}
|
|
20
|
+
// --------------------------------------------------------
|
|
21
|
+
// remember() — Store a new memory
|
|
22
|
+
// --------------------------------------------------------
|
|
23
|
+
remember(input) {
|
|
24
|
+
// Accept a plain string for convenience
|
|
25
|
+
const parsed = typeof input === 'string'
|
|
26
|
+
? RememberInputSchema.parse({ content: input })
|
|
27
|
+
: RememberInputSchema.parse(input);
|
|
28
|
+
// Auto-extract entities and topics if not provided
|
|
29
|
+
if (parsed.entities.length === 0 && parsed.topics.length === 0) {
|
|
30
|
+
const extracted = extract(parsed.content);
|
|
31
|
+
if (parsed.entities.length === 0)
|
|
32
|
+
parsed.entities = extracted.entities;
|
|
33
|
+
if (parsed.topics.length === 0)
|
|
34
|
+
parsed.topics = extracted.topics;
|
|
35
|
+
// Only use suggested salience if user didn't set one (default is 0.5)
|
|
36
|
+
if (parsed.salience === 0.5)
|
|
37
|
+
parsed.salience = extracted.suggestedSalience;
|
|
38
|
+
}
|
|
39
|
+
// Auto-set source metadata from vault config
|
|
40
|
+
if (!parsed.source) {
|
|
41
|
+
parsed.source = { type: 'conversation' };
|
|
42
|
+
}
|
|
43
|
+
if (this.config.agentId && !parsed.source.agentId) {
|
|
44
|
+
parsed.source.agentId = this.config.agentId;
|
|
45
|
+
}
|
|
46
|
+
if (this.config.sessionId && !parsed.source.sessionId) {
|
|
47
|
+
parsed.source.sessionId = this.config.sessionId;
|
|
48
|
+
}
|
|
49
|
+
const memory = this.store.createMemory(parsed);
|
|
50
|
+
// Queue embedding computation (non-blocking but tracked)
|
|
51
|
+
// Also checks for near-duplicates after embedding is computed.
|
|
52
|
+
if (this.embedder) {
|
|
53
|
+
const p = this.computeAndStoreEmbedding(memory.id, memory.content)
|
|
54
|
+
.then(() => {
|
|
55
|
+
// Dedup check: if a very similar memory already exists, merge instead of keeping both
|
|
56
|
+
this.dedup(memory);
|
|
57
|
+
})
|
|
58
|
+
.catch(err => {
|
|
59
|
+
console.warn(`Failed to compute embedding for ${memory.id}:`, err);
|
|
60
|
+
})
|
|
61
|
+
.finally(() => {
|
|
62
|
+
this.pendingEmbeddings.delete(p);
|
|
63
|
+
});
|
|
64
|
+
this.pendingEmbeddings.add(p);
|
|
65
|
+
}
|
|
66
|
+
return memory;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Dedup: after storing a memory and its embedding, check if a near-identical
|
|
70
|
+
* memory already exists. If so, keep the better one (higher salience/confidence,
|
|
71
|
+
* or newer if semantic) and supersede the other.
|
|
72
|
+
*
|
|
73
|
+
* Threshold: cosine distance <= 0.08 (similarity >= 0.92) = near-duplicate.
|
|
74
|
+
* Only dedup within the same type (don't merge episodic into semantic).
|
|
75
|
+
*/
|
|
76
|
+
dedup(memory) {
|
|
77
|
+
// Don't dedup consolidation outputs — they're intentional
|
|
78
|
+
if (memory.source?.type === 'consolidation')
|
|
79
|
+
return;
|
|
80
|
+
try {
|
|
81
|
+
const similar = this.store.findSimilar(this.store.getEmbedding(memory.id) ?? [], 0.08, // cosine distance threshold — very tight, ~92% similarity
|
|
82
|
+
5);
|
|
83
|
+
for (const match of similar) {
|
|
84
|
+
if (match.memoryId === memory.id)
|
|
85
|
+
continue; // skip self
|
|
86
|
+
const existing = this.store.getMemoryDirect(match.memoryId);
|
|
87
|
+
if (!existing)
|
|
88
|
+
continue;
|
|
89
|
+
if (existing.status !== 'active')
|
|
90
|
+
continue;
|
|
91
|
+
if (existing.type !== memory.type)
|
|
92
|
+
continue; // only dedup same type
|
|
93
|
+
// We have a near-duplicate. Keep the one with higher salience,
|
|
94
|
+
// or if equal, keep the newer one (more up-to-date).
|
|
95
|
+
const keepNew = memory.salience >= existing.salience;
|
|
96
|
+
const supersededId = keepNew ? existing.id : memory.id;
|
|
97
|
+
const keptId = keepNew ? memory.id : existing.id;
|
|
98
|
+
this.store.updateStatus(supersededId, 'superseded');
|
|
99
|
+
// Create a supersedes edge so the graph tracks the lineage
|
|
100
|
+
this.store.createEdge(keptId, supersededId, 'supersedes', 0.8);
|
|
101
|
+
// Merge: boost the kept memory's salience slightly from the duplicate
|
|
102
|
+
const kept = this.store.getMemoryDirect(keptId);
|
|
103
|
+
if (kept && kept.salience < 1.0) {
|
|
104
|
+
this.store.updateMemory(keptId, {
|
|
105
|
+
salience: Math.min(1.0, kept.salience + 0.05),
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
break; // Only process one duplicate match
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// Dedup is best-effort; don't break remember() if it fails
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/** Compute embedding and store it — can be awaited if needed */
|
|
116
|
+
async computeAndStoreEmbedding(memoryId, content) {
|
|
117
|
+
if (!this.embedder)
|
|
118
|
+
return;
|
|
119
|
+
const embedding = await this.embedder.embed(content);
|
|
120
|
+
this.store.storeEmbedding(memoryId, embedding);
|
|
121
|
+
}
|
|
122
|
+
/** Batch compute embeddings for all memories missing them */
|
|
123
|
+
async backfillEmbeddings() {
|
|
124
|
+
if (!this.embedder)
|
|
125
|
+
return 0;
|
|
126
|
+
const allMemories = this.store.exportAll().memories;
|
|
127
|
+
let count = 0;
|
|
128
|
+
// Process in batches of 50
|
|
129
|
+
for (let i = 0; i < allMemories.length; i += 50) {
|
|
130
|
+
const batch = allMemories.slice(i, i + 50);
|
|
131
|
+
const texts = batch.map(m => m.content);
|
|
132
|
+
const embeddings = await this.embedder.embedBatch(texts);
|
|
133
|
+
for (let j = 0; j < batch.length; j++) {
|
|
134
|
+
this.store.storeEmbedding(batch[j].id, embeddings[j]);
|
|
135
|
+
count++;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return count;
|
|
139
|
+
}
|
|
140
|
+
// --------------------------------------------------------
|
|
141
|
+
// recall() — Retrieve relevant memories for a context
|
|
142
|
+
// --------------------------------------------------------
|
|
143
|
+
async recall(input) {
|
|
144
|
+
const parsed = typeof input === 'string'
|
|
145
|
+
? RecallInputSchema.parse({ context: input })
|
|
146
|
+
: RecallInputSchema.parse(input);
|
|
147
|
+
const candidates = new Map();
|
|
148
|
+
// ── Phase 0: Auto-extract entities and topics from query ──
|
|
149
|
+
// If the caller didn't provide explicit entities/topics,
|
|
150
|
+
// extract them from the context string so entity/topic
|
|
151
|
+
// retrieval actually fires. Try LLM extraction first if available,
|
|
152
|
+
// then fall back to rule-based extraction.
|
|
153
|
+
if ((!parsed.entities || parsed.entities.length === 0) ||
|
|
154
|
+
(!parsed.topics || parsed.topics.length === 0)) {
|
|
155
|
+
// Try LLM extraction if vault has LLM config
|
|
156
|
+
if (this.config.llm && (!parsed.entities || parsed.entities.length === 0) && (!parsed.topics || parsed.topics.length === 0)) {
|
|
157
|
+
try {
|
|
158
|
+
const llmExtracted = await this.extractWithLLM(parsed.context);
|
|
159
|
+
if (!parsed.entities || parsed.entities.length === 0) {
|
|
160
|
+
parsed.entities = llmExtracted.entities;
|
|
161
|
+
}
|
|
162
|
+
if (!parsed.topics || parsed.topics.length === 0) {
|
|
163
|
+
parsed.topics = llmExtracted.topics;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
catch (err) {
|
|
167
|
+
// LLM extraction failed — fall back to rule-based
|
|
168
|
+
console.warn('LLM extraction failed, falling back to rule-based:', err);
|
|
169
|
+
const extracted = extract(parsed.context);
|
|
170
|
+
if (!parsed.entities || parsed.entities.length === 0) {
|
|
171
|
+
parsed.entities = extracted.entities;
|
|
172
|
+
}
|
|
173
|
+
if (!parsed.topics || parsed.topics.length === 0) {
|
|
174
|
+
parsed.topics = extracted.topics;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
// No LLM config or entities/topics already provided — use rule-based
|
|
180
|
+
const extracted = extract(parsed.context);
|
|
181
|
+
if (!parsed.entities || parsed.entities.length === 0) {
|
|
182
|
+
parsed.entities = extracted.entities;
|
|
183
|
+
}
|
|
184
|
+
if (!parsed.topics || parsed.topics.length === 0) {
|
|
185
|
+
parsed.topics = extracted.topics;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// ── Phase 0b: Aggregation detection ──
|
|
190
|
+
const aggregationPatterns = [
|
|
191
|
+
/\ball\b.*\b(commitments?|promises?|pending|outstanding)\b/i,
|
|
192
|
+
/\b(pending|outstanding|unfulfilled)\b.*\b(commitments?|promises?|tasks?)\b/i,
|
|
193
|
+
/\b(corrected|updated|changed|revised)\b/i,
|
|
194
|
+
/\bkey\s+(metrics?|numbers?|stats?|statistics?|KPIs?)\b/i,
|
|
195
|
+
/\bevery\b|\blist\s+(of|all)\b|\bcomplete\s+list\b/i,
|
|
196
|
+
];
|
|
197
|
+
const isAggregation = aggregationPatterns.some(p => p.test(parsed.context));
|
|
198
|
+
if (isAggregation) {
|
|
199
|
+
const aggTopics = parsed.topics ?? [];
|
|
200
|
+
for (const topic of aggTopics) {
|
|
201
|
+
const topicMemories = this.store.getByTopic(topic, 50);
|
|
202
|
+
for (const mem of topicMemories) {
|
|
203
|
+
this.addCandidate(candidates, mem, 0.3);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (/commitment|pending|promise|outstanding|unfulfilled/i.test(parsed.context)) {
|
|
207
|
+
const pendingMemories = this.store.getByStatus('pending', 50);
|
|
208
|
+
for (const mem of pendingMemories) {
|
|
209
|
+
this.addCandidate(candidates, mem, 0.4);
|
|
210
|
+
}
|
|
211
|
+
const fulfilledMemories = this.store.getByStatus('fulfilled', 50);
|
|
212
|
+
for (const mem of fulfilledMemories) {
|
|
213
|
+
this.addCandidate(candidates, mem, 0.3);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (/correct|update|change|revis|wrong/i.test(parsed.context)) {
|
|
217
|
+
const supersededMemories = this.store.getByStatus('superseded', 30);
|
|
218
|
+
for (const mem of supersededMemories) {
|
|
219
|
+
this.addCandidate(candidates, mem, 0.35);
|
|
220
|
+
}
|
|
221
|
+
const correctionMemories = this.store.getByTopic('correction', 30);
|
|
222
|
+
for (const mem of correctionMemories) {
|
|
223
|
+
this.addCandidate(candidates, mem, 0.35);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (/metric|number|stat|KPI|measure/i.test(parsed.context)) {
|
|
227
|
+
const metricsMemories = this.store.getByTopic('metrics', 50);
|
|
228
|
+
for (const mem of metricsMemories) {
|
|
229
|
+
this.addCandidate(candidates, mem, 0.35);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
parsed.limit = Math.max(parsed.limit, 30);
|
|
233
|
+
}
|
|
234
|
+
// ── Phase 1: Direct retrieval (seed memories) ──────────
|
|
235
|
+
// ── Phase 1 Strategy ──
|
|
236
|
+
// Vector search is the primary retrieval signal — it finds
|
|
237
|
+
// what's semantically relevant to the query. Entity/topic
|
|
238
|
+
// matching acts as a secondary boost, not a primary retriever,
|
|
239
|
+
// because common entities (e.g. "Thomas" in 100+ memories)
|
|
240
|
+
// flood the candidate pool with noise if scored too high.
|
|
241
|
+
// 1. Semantic search via embeddings (PRIMARY — highest signal)
|
|
242
|
+
if (this.embedder && this.store.hasVectorSearch()) {
|
|
243
|
+
try {
|
|
244
|
+
const queryEmbedding = await this.embedder.embed(parsed.context);
|
|
245
|
+
const vectorResults = this.store.searchByVector(queryEmbedding, 50);
|
|
246
|
+
for (const vr of vectorResults) {
|
|
247
|
+
const mem = this.store.getMemoryDirect(vr.memoryId);
|
|
248
|
+
if (mem) {
|
|
249
|
+
// Use cosine similarity (1 - distance) as primary score
|
|
250
|
+
const similarity = Math.max(0, 1 - vr.distance);
|
|
251
|
+
this.addCandidate(candidates, mem, similarity);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
// Vector search failed — keyword search becomes primary
|
|
257
|
+
this.keywordSearch(parsed.context, candidates, 0.4);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
// No embeddings available — keyword is primary
|
|
262
|
+
this.keywordSearch(parsed.context, candidates, 0.4);
|
|
263
|
+
}
|
|
264
|
+
// 1b. Keyword search (ALWAYS runs as supplementary signal)
|
|
265
|
+
// Catches exact term matches that embeddings might miss —
|
|
266
|
+
// e.g. "competitors" in a query matching "competitors" in content.
|
|
267
|
+
this.keywordSearch(parsed.context, candidates, 0.2);
|
|
268
|
+
// 2. Entity-based retrieval (SECONDARY — boost, not flood)
|
|
269
|
+
// Pull more candidates for entity matches but weight by type:
|
|
270
|
+
// semantic memories are higher signal (consolidated knowledge)
|
|
271
|
+
// than raw episodic entries.
|
|
272
|
+
if (parsed.entities && parsed.entities.length > 0) {
|
|
273
|
+
for (const entity of parsed.entities) {
|
|
274
|
+
const memories = this.store.getByEntity(entity, 20);
|
|
275
|
+
for (const mem of memories) {
|
|
276
|
+
// Scale entity base score: fewer results = higher confidence each is relevant
|
|
277
|
+
const baseScore = memories.length <= 5 ? 0.25 : memories.length <= 15 ? 0.15 : 0.1;
|
|
278
|
+
// Semantic memories from entity match get a bonus — they're distilled facts
|
|
279
|
+
const typeBonus = mem.type === 'semantic' ? 0.1 : 0;
|
|
280
|
+
this.addCandidate(candidates, mem, baseScore + typeBonus);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// 3. Topic-based retrieval (SECONDARY)
|
|
285
|
+
if (parsed.topics && parsed.topics.length > 0) {
|
|
286
|
+
for (const topic of parsed.topics) {
|
|
287
|
+
const memories = this.store.getByTopic(topic, 10);
|
|
288
|
+
const topicScore = memories.length <= 3 ? 0.2 : 0.08;
|
|
289
|
+
for (const mem of memories) {
|
|
290
|
+
this.addCandidate(candidates, mem, topicScore);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
// 4. Recent memories (light recency signal)
|
|
295
|
+
const recent = this.store.getRecent(5);
|
|
296
|
+
for (const mem of recent) {
|
|
297
|
+
this.addCandidate(candidates, mem, 0.05);
|
|
298
|
+
}
|
|
299
|
+
// ── Phase 2: Spreading activation ──────────────────────
|
|
300
|
+
// Take the seeds from Phase 1 and let activation cascade
|
|
301
|
+
// through the memory graph. This is what makes recall feel
|
|
302
|
+
// like memory instead of search.
|
|
303
|
+
if (parsed.spread && candidates.size > 0) {
|
|
304
|
+
this.spreadActivation(candidates, {
|
|
305
|
+
maxHops: parsed.spreadHops,
|
|
306
|
+
decay: parsed.spreadDecay,
|
|
307
|
+
minActivation: parsed.spreadMinActivation,
|
|
308
|
+
entityHops: parsed.spreadEntityHops,
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
// ── Phase 3: Filter, score, rank ───────────────────────
|
|
312
|
+
// 5. Filter out superseded/archived memories first
|
|
313
|
+
const showSuperseded = isAggregation && /correct|update|change|revis/i.test(parsed.context);
|
|
314
|
+
let results = [...candidates.values()].filter(r => r.memory.status !== 'archived' &&
|
|
315
|
+
(r.memory.status !== 'superseded' || showSuperseded));
|
|
316
|
+
// Type filter
|
|
317
|
+
if (parsed.types && parsed.types.length > 0) {
|
|
318
|
+
results = results.filter(r => parsed.types.includes(r.memory.type));
|
|
319
|
+
}
|
|
320
|
+
// 6. Apply minimum thresholds
|
|
321
|
+
results = results.filter(r => r.memory.salience >= parsed.minSalience &&
|
|
322
|
+
r.memory.confidence >= parsed.minConfidence);
|
|
323
|
+
// 7. Temporal focus
|
|
324
|
+
if (parsed.temporalFocus === 'recent') {
|
|
325
|
+
const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
|
326
|
+
results = results.filter(r => r.memory.createdAt >= oneWeekAgo);
|
|
327
|
+
}
|
|
328
|
+
// 8. Score with salience, stability, and type weighting
|
|
329
|
+
// Semantic memories with high stability are the "core knowledge" —
|
|
330
|
+
// they should outrank noisy episodic results when the vector
|
|
331
|
+
// similarity scores are close. Without this, basic factual queries
|
|
332
|
+
// like "What is Thomas's job?" get buried under episodic noise.
|
|
333
|
+
for (const r of results) {
|
|
334
|
+
const cappedStability = Math.min(r.memory.stability, 3.0);
|
|
335
|
+
// Base weight from salience and stability
|
|
336
|
+
const salienceBoost = r.memory.salience * 0.25;
|
|
337
|
+
const stabilityBoost = cappedStability * 0.1;
|
|
338
|
+
// Type bonus: consolidated semantic memories are higher-signal
|
|
339
|
+
// than raw episodic entries for factual queries
|
|
340
|
+
const typeBonus = r.memory.type === 'semantic' ? 0.25 : 0;
|
|
341
|
+
// Confidence bonus: high-confidence memories are more reliable
|
|
342
|
+
const confidenceBonus = r.memory.confidence * 0.05;
|
|
343
|
+
// Superseded/archived penalty: shouldn't appear in results
|
|
344
|
+
const statusPenalty = (r.memory.status === 'superseded' || r.memory.status === 'archived') ? 0.5 : 0;
|
|
345
|
+
r.score = r.score * (0.5 + salienceBoost + stabilityBoost + typeBonus + confidenceBonus) - statusPenalty;
|
|
346
|
+
}
|
|
347
|
+
// 9. Sort by score and return top N
|
|
348
|
+
results.sort((a, b) => b.score - a.score);
|
|
349
|
+
// Mark accessed (only the returned results, not traversal noise)
|
|
350
|
+
const topResults = results.slice(0, parsed.limit);
|
|
351
|
+
for (const r of topResults) {
|
|
352
|
+
this.store.getMemory(r.memory.id); // Triggers access count + stability update
|
|
353
|
+
}
|
|
354
|
+
return topResults.map(r => r.memory);
|
|
355
|
+
}
|
|
356
|
+
// --------------------------------------------------------
|
|
357
|
+
// Spreading Activation — The cascade that makes recall
|
|
358
|
+
// feel like memory instead of search.
|
|
359
|
+
//
|
|
360
|
+
// Algorithm:
|
|
361
|
+
// 1. Seeds come in with initial activation scores from Phase 1
|
|
362
|
+
// 2. For each hop:
|
|
363
|
+
// a. Collect all edges from currently active memories
|
|
364
|
+
// b. For each neighbor: activation = parent_activation × edge_strength × decay
|
|
365
|
+
// c. Also spread via shared entities (implicit edges)
|
|
366
|
+
// d. Add/boost neighbor in candidate pool
|
|
367
|
+
// 3. Stop when activation falls below threshold or max hops reached
|
|
368
|
+
//
|
|
369
|
+
// This is why querying "Thomas" can surface his marathon training
|
|
370
|
+
// schedule even if you only asked about his work preferences.
|
|
371
|
+
// --------------------------------------------------------
|
|
372
|
+
spreadActivation(candidates, opts) {
|
|
373
|
+
// Current frontier: memory IDs and their activation level
|
|
374
|
+
let frontier = new Map();
|
|
375
|
+
// Initialize frontier from current candidates
|
|
376
|
+
for (const [id, { score }] of candidates) {
|
|
377
|
+
frontier.set(id, score);
|
|
378
|
+
}
|
|
379
|
+
const visited = new Set(frontier.keys());
|
|
380
|
+
for (let hop = 0; hop < opts.maxHops; hop++) {
|
|
381
|
+
const nextFrontier = new Map();
|
|
382
|
+
const frontierIds = [...frontier.keys()];
|
|
383
|
+
if (frontierIds.length === 0)
|
|
384
|
+
break;
|
|
385
|
+
// ── Edge-based spreading ──
|
|
386
|
+
const edges = this.store.getEdgesForMemories(frontierIds);
|
|
387
|
+
for (const edge of edges) {
|
|
388
|
+
const parentId = frontier.has(edge.sourceId) ? edge.sourceId : edge.targetId;
|
|
389
|
+
const neighborId = edge.sourceId === parentId ? edge.targetId : edge.sourceId;
|
|
390
|
+
const parentActivation = frontier.get(parentId) ?? 0;
|
|
391
|
+
// Activation = parent × edge_strength × decay × edge_type_weight
|
|
392
|
+
const typeWeight = this.edgeTypeWeight(edge.type);
|
|
393
|
+
const activation = parentActivation * edge.strength * opts.decay * typeWeight;
|
|
394
|
+
if (activation < opts.minActivation)
|
|
395
|
+
continue;
|
|
396
|
+
// Accumulate activation (multiple paths can reinforce)
|
|
397
|
+
const existing = nextFrontier.get(neighborId) ?? 0;
|
|
398
|
+
nextFrontier.set(neighborId, Math.min(existing + activation, 1.0));
|
|
399
|
+
}
|
|
400
|
+
// ── Entity-based spreading (implicit edges) ──
|
|
401
|
+
// Memories that share entities are implicitly connected.
|
|
402
|
+
// This is crucial when the explicit graph is sparse.
|
|
403
|
+
if (opts.entityHops) {
|
|
404
|
+
for (const id of frontierIds) {
|
|
405
|
+
const parentActivation = frontier.get(id) ?? 0;
|
|
406
|
+
const coEntities = this.store.getCoEntityMemories(id, 10);
|
|
407
|
+
for (const { memory: neighbor, sharedEntities } of coEntities) {
|
|
408
|
+
if (visited.has(neighbor.id))
|
|
409
|
+
continue;
|
|
410
|
+
// More shared entities = stronger implicit connection
|
|
411
|
+
const implicitStrength = Math.min(sharedEntities.length * 0.3, 0.9);
|
|
412
|
+
const activation = parentActivation * implicitStrength * opts.decay;
|
|
413
|
+
if (activation < opts.minActivation)
|
|
414
|
+
continue;
|
|
415
|
+
const existing = nextFrontier.get(neighbor.id) ?? 0;
|
|
416
|
+
nextFrontier.set(neighbor.id, Math.min(existing + activation, 1.0));
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
// Load activated memories and add to candidates
|
|
421
|
+
const newIds = [...nextFrontier.keys()].filter(id => !visited.has(id));
|
|
422
|
+
if (newIds.length === 0 && [...nextFrontier.keys()].every(id => visited.has(id)))
|
|
423
|
+
break;
|
|
424
|
+
const newMemories = this.store.getMemoriesDirect(newIds);
|
|
425
|
+
const memoryMap = new Map(newMemories.map(m => [m.id, m]));
|
|
426
|
+
for (const [id, activation] of nextFrontier) {
|
|
427
|
+
const memory = memoryMap.get(id) ?? candidates.get(id)?.memory;
|
|
428
|
+
if (!memory)
|
|
429
|
+
continue;
|
|
430
|
+
// Tag that this came from spreading (for debugging/eval)
|
|
431
|
+
// Use a reduced weight — spread results shouldn't dominate direct hits
|
|
432
|
+
const spreadWeight = 0.6;
|
|
433
|
+
this.addCandidate(candidates, memory, activation * spreadWeight);
|
|
434
|
+
visited.add(id);
|
|
435
|
+
}
|
|
436
|
+
// Next hop starts from newly activated memories
|
|
437
|
+
frontier = new Map();
|
|
438
|
+
for (const [id, activation] of nextFrontier) {
|
|
439
|
+
if (activation >= opts.minActivation) {
|
|
440
|
+
frontier.set(id, activation);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
// --------------------------------------------------------
|
|
446
|
+
// Edge type weights — how strongly different relationship
|
|
447
|
+
// types propagate activation.
|
|
448
|
+
// --------------------------------------------------------
|
|
449
|
+
edgeTypeWeight(type) {
|
|
450
|
+
switch (type) {
|
|
451
|
+
case 'supports': return 0.9; // Strong: supporting evidence propagates well
|
|
452
|
+
case 'elaborates': return 0.85; // Strong: detail enriches context
|
|
453
|
+
case 'causes': return 0.8; // Causal chains are highly relevant
|
|
454
|
+
case 'caused_by': return 0.8;
|
|
455
|
+
case 'part_of': return 0.75; // Part-whole relationships matter
|
|
456
|
+
case 'instance_of': return 0.7; // Specific→general is useful
|
|
457
|
+
case 'supersedes': return 0.6; // Updated info still connects
|
|
458
|
+
case 'associated_with': return 0.5; // Weak but valid
|
|
459
|
+
case 'temporal_next': return 0.4; // Temporal sequence is loose
|
|
460
|
+
case 'derived_from': return 0.7; // Consolidation lineage
|
|
461
|
+
case 'contradicts': return 0.3; // Contradictions are relevant but shouldn't dominate
|
|
462
|
+
default: return 0.5;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
// --------------------------------------------------------
|
|
466
|
+
// forget() — Explicitly remove or decay a memory
|
|
467
|
+
// --------------------------------------------------------
|
|
468
|
+
forget(id, hard = false) {
|
|
469
|
+
if (hard) {
|
|
470
|
+
this.store.deleteMemory(id);
|
|
471
|
+
}
|
|
472
|
+
else {
|
|
473
|
+
// Soft forget: drastically reduce stability
|
|
474
|
+
this.store.updateMemory(id, { salience: 0 });
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
// --------------------------------------------------------
|
|
478
|
+
// connect() — Create a relationship between memories
|
|
479
|
+
// --------------------------------------------------------
|
|
480
|
+
connect(sourceId, targetId, type, strength = 0.5) {
|
|
481
|
+
return this.store.createEdge(sourceId, targetId, type, strength);
|
|
482
|
+
}
|
|
483
|
+
// --------------------------------------------------------
|
|
484
|
+
// neighbors() — Get related memories via graph traversal
|
|
485
|
+
// --------------------------------------------------------
|
|
486
|
+
neighbors(memoryId, depth = 1) {
|
|
487
|
+
return this.store.getNeighbors(memoryId, depth);
|
|
488
|
+
}
|
|
489
|
+
// --------------------------------------------------------
|
|
490
|
+
// consolidate() — The magic: turn episodes into knowledge
|
|
491
|
+
// --------------------------------------------------------
|
|
492
|
+
async consolidate() {
|
|
493
|
+
const startedAt = new Date().toISOString();
|
|
494
|
+
// Get recent unconsolidated episodes
|
|
495
|
+
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
|
496
|
+
const episodes = this.store.getEpisodicSince(oneDayAgo);
|
|
497
|
+
let semanticCreated = 0;
|
|
498
|
+
let semanticUpdated = 0;
|
|
499
|
+
let entitiesDiscovered = 0;
|
|
500
|
+
let connectionsFormed = 0;
|
|
501
|
+
let contradictionsFound = 0;
|
|
502
|
+
if (this.config.llm) {
|
|
503
|
+
// LLM-powered consolidation
|
|
504
|
+
const result = await this.llmConsolidate(episodes);
|
|
505
|
+
semanticCreated = result.semanticCreated;
|
|
506
|
+
semanticUpdated = result.semanticUpdated;
|
|
507
|
+
entitiesDiscovered = result.entitiesDiscovered;
|
|
508
|
+
connectionsFormed = result.connectionsFormed;
|
|
509
|
+
contradictionsFound = result.contradictionsFound;
|
|
510
|
+
}
|
|
511
|
+
else {
|
|
512
|
+
// Rule-based consolidation (no LLM required)
|
|
513
|
+
const result = this.ruleBasedConsolidate(episodes);
|
|
514
|
+
semanticCreated = result.semanticCreated;
|
|
515
|
+
entitiesDiscovered = result.entitiesDiscovered;
|
|
516
|
+
connectionsFormed = result.connectionsFormed;
|
|
517
|
+
}
|
|
518
|
+
// Apply decay
|
|
519
|
+
const decayed = this.store.applyDecay(this.config.decay?.halfLifeHours ?? 168);
|
|
520
|
+
// Archive deeply decayed memories
|
|
521
|
+
const archived = this.store.getDecayedMemories(this.config.decay?.archiveThreshold ?? 0.05);
|
|
522
|
+
for (const mem of archived) {
|
|
523
|
+
this.store.deleteMemory(mem.id); // TODO: move to cold storage instead of deleting
|
|
524
|
+
}
|
|
525
|
+
const report = {
|
|
526
|
+
startedAt,
|
|
527
|
+
completedAt: new Date().toISOString(),
|
|
528
|
+
episodesProcessed: episodes.length,
|
|
529
|
+
semanticMemoriesCreated: semanticCreated,
|
|
530
|
+
semanticMemoriesUpdated: semanticUpdated,
|
|
531
|
+
entitiesDiscovered,
|
|
532
|
+
connectionsFormed,
|
|
533
|
+
contradictionsFound,
|
|
534
|
+
memoriesDecayed: decayed,
|
|
535
|
+
memoriesArchived: archived.length,
|
|
536
|
+
};
|
|
537
|
+
// Store the consolidation report as a memory itself
|
|
538
|
+
this.remember({
|
|
539
|
+
content: `Consolidation completed: processed ${episodes.length} episodes, created ${semanticCreated} semantic memories, discovered ${entitiesDiscovered} entities, formed ${connectionsFormed} connections, decayed ${decayed} memories.`,
|
|
540
|
+
type: 'procedural',
|
|
541
|
+
topics: ['meta', 'consolidation'],
|
|
542
|
+
salience: 0.3,
|
|
543
|
+
source: { type: 'consolidation' },
|
|
544
|
+
});
|
|
545
|
+
return report;
|
|
546
|
+
}
|
|
547
|
+
// --------------------------------------------------------
|
|
548
|
+
// briefing() — Structured context summary for session start.
|
|
549
|
+
// This is the MEMORY.md replacement: instead of reading a
|
|
550
|
+
// flat file, an agent calls POST /v1/briefing and gets a
|
|
551
|
+
// curated knowledge snapshot.
|
|
552
|
+
// --------------------------------------------------------
|
|
553
|
+
async briefing(context = '', limit = 20) {
|
|
554
|
+
// 1. High-salience semantic memories (key facts)
|
|
555
|
+
const allSemantic = this.store.getByType('semantic', 100);
|
|
556
|
+
const keyFacts = allSemantic
|
|
557
|
+
.filter(m => m.salience >= 0.4 && m.status === 'active')
|
|
558
|
+
.sort((a, b) => b.salience - a.salience)
|
|
559
|
+
.slice(0, limit)
|
|
560
|
+
.map(m => ({ content: m.content, salience: m.salience, entities: m.entities }));
|
|
561
|
+
// 2. Active commitments (pending status)
|
|
562
|
+
const allMemories = this.store.exportAll().memories;
|
|
563
|
+
const activeCommitments = allMemories
|
|
564
|
+
.filter(m => m.status === 'pending')
|
|
565
|
+
.sort((a, b) => b.salience - a.salience)
|
|
566
|
+
.slice(0, 10)
|
|
567
|
+
.map(m => ({ content: m.content, status: m.status, entities: m.entities }));
|
|
568
|
+
// 3. Recent activity (last 24h episodes)
|
|
569
|
+
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
|
570
|
+
const recentActivity = this.store.getEpisodicSince(oneDayAgo, 10)
|
|
571
|
+
.map(m => ({ content: m.content, when: m.createdAt }));
|
|
572
|
+
// 4. Top entities
|
|
573
|
+
const topEntities = this.entities()
|
|
574
|
+
.slice(0, 10)
|
|
575
|
+
.map(e => ({ name: e.name, type: e.type, memoryCount: e.memoryCount }));
|
|
576
|
+
// 5. Contradictions
|
|
577
|
+
const contradictions = this.contradictions(5)
|
|
578
|
+
.map(c => ({ a: c.memoryA.content, b: c.memoryB.content }));
|
|
579
|
+
// 6. If context is provided, do a spreading-activation recall and weave in results
|
|
580
|
+
let contextualMemories = [];
|
|
581
|
+
if (context.trim()) {
|
|
582
|
+
const recalled = await this.recall({ context, limit: 5, spread: true });
|
|
583
|
+
contextualMemories = recalled.map(m => m.content);
|
|
584
|
+
}
|
|
585
|
+
// 7. Build summary
|
|
586
|
+
const stats = this.stats();
|
|
587
|
+
const summaryParts = [];
|
|
588
|
+
summaryParts.push(`Vault: ${stats.total} memories (${stats.semantic} semantic, ${stats.episodic} episodic, ${stats.procedural} procedural), ${stats.entities} entities.`);
|
|
589
|
+
if (activeCommitments.length > 0) {
|
|
590
|
+
summaryParts.push(`${activeCommitments.length} pending commitment(s).`);
|
|
591
|
+
}
|
|
592
|
+
if (contradictions.length > 0) {
|
|
593
|
+
summaryParts.push(`${contradictions.length} unresolved contradiction(s).`);
|
|
594
|
+
}
|
|
595
|
+
if (contextualMemories.length > 0) {
|
|
596
|
+
summaryParts.push(`Context-relevant: ${contextualMemories.join(' | ')}`);
|
|
597
|
+
}
|
|
598
|
+
return {
|
|
599
|
+
summary: summaryParts.join(' '),
|
|
600
|
+
keyFacts,
|
|
601
|
+
activeCommitments,
|
|
602
|
+
recentActivity,
|
|
603
|
+
topEntities,
|
|
604
|
+
contradictions,
|
|
605
|
+
stats,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
// --------------------------------------------------------
|
|
609
|
+
// contradictions() — Find unresolved conflicts in the graph.
|
|
610
|
+
// No competitor has this. It's a real differentiator.
|
|
611
|
+
//
|
|
612
|
+
// Checks:
|
|
613
|
+
// 1. Explicit 'contradicts' edges in the graph
|
|
614
|
+
// 2. Status conflicts (superseded memories with active successors)
|
|
615
|
+
// 3. Entity-scoped content conflicts (LLM-powered if available)
|
|
616
|
+
// --------------------------------------------------------
|
|
617
|
+
contradictions(limit = 50) {
|
|
618
|
+
const results = [];
|
|
619
|
+
// 1. Explicit contradiction edges
|
|
620
|
+
const allExport = this.store.exportAll();
|
|
621
|
+
const memoryMap = new Map(allExport.memories.map(m => [m.id, m]));
|
|
622
|
+
for (const edge of allExport.edges) {
|
|
623
|
+
if (edge.type === 'contradicts') {
|
|
624
|
+
const a = memoryMap.get(edge.sourceId);
|
|
625
|
+
const b = memoryMap.get(edge.targetId);
|
|
626
|
+
if (a && b) {
|
|
627
|
+
results.push({
|
|
628
|
+
memoryA: a,
|
|
629
|
+
memoryB: b,
|
|
630
|
+
type: 'explicit_edge',
|
|
631
|
+
description: `Explicit contradiction (edge strength: ${edge.strength.toFixed(2)})`,
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
if (edge.type === 'supersedes') {
|
|
636
|
+
const newer = memoryMap.get(edge.sourceId);
|
|
637
|
+
const older = memoryMap.get(edge.targetId);
|
|
638
|
+
if (newer && older && older.status === 'active') {
|
|
639
|
+
results.push({
|
|
640
|
+
memoryA: newer,
|
|
641
|
+
memoryB: older,
|
|
642
|
+
type: 'superseded_conflict',
|
|
643
|
+
description: `"${newer.summary}" supersedes "${older.summary}" but older is still marked active`,
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
// 2. Find potential entity-scoped conflicts
|
|
649
|
+
// Group semantic memories by entity, look for opposing claims
|
|
650
|
+
const entityMemories = new Map();
|
|
651
|
+
for (const mem of allExport.memories) {
|
|
652
|
+
if (mem.type !== 'semantic' || mem.status !== 'active')
|
|
653
|
+
continue;
|
|
654
|
+
for (const entity of mem.entities) {
|
|
655
|
+
const list = entityMemories.get(entity) ?? [];
|
|
656
|
+
list.push(mem);
|
|
657
|
+
entityMemories.set(entity, list);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
// Simple heuristic: if two semantic memories about the same entity
|
|
661
|
+
// have conflicting signals (negation words, opposite qualifiers)
|
|
662
|
+
const negationPatterns = [
|
|
663
|
+
/\bnot\b/i, /\bnever\b/i, /\bno longer\b/i, /\bstopped\b/i,
|
|
664
|
+
/\bwon't\b/i, /\bdoesn't\b/i, /\bisn't\b/i, /\bwasn't\b/i,
|
|
665
|
+
/\bhates?\b/i, /\bdislikes?\b/i, /\bavoids?\b/i,
|
|
666
|
+
];
|
|
667
|
+
const affirmationPatterns = [
|
|
668
|
+
/\balways\b/i, /\bloves?\b/i, /\bprefers?\b/i, /\bfavorite\b/i,
|
|
669
|
+
/\bregularly\b/i, /\bevery\b/i, /\benjoying\b/i,
|
|
670
|
+
];
|
|
671
|
+
for (const [entity, mems] of entityMemories) {
|
|
672
|
+
if (mems.length < 2)
|
|
673
|
+
continue;
|
|
674
|
+
for (let i = 0; i < mems.length && results.length < limit; i++) {
|
|
675
|
+
for (let j = i + 1; j < mems.length && results.length < limit; j++) {
|
|
676
|
+
const a = mems[i];
|
|
677
|
+
const b = mems[j];
|
|
678
|
+
const aHasNeg = negationPatterns.some(p => p.test(a.content));
|
|
679
|
+
const bHasAff = affirmationPatterns.some(p => p.test(b.content));
|
|
680
|
+
const aHasAff = affirmationPatterns.some(p => p.test(a.content));
|
|
681
|
+
const bHasNeg = negationPatterns.some(p => p.test(b.content));
|
|
682
|
+
if ((aHasNeg && bHasAff) || (aHasAff && bHasNeg)) {
|
|
683
|
+
// Check they're about a similar topic (share >1 entity or topic)
|
|
684
|
+
const sharedEntities = a.entities.filter(e => b.entities.includes(e));
|
|
685
|
+
const sharedTopics = a.topics.filter(t => b.topics.includes(t));
|
|
686
|
+
if (sharedEntities.length + sharedTopics.length >= 1) {
|
|
687
|
+
results.push({
|
|
688
|
+
memoryA: a,
|
|
689
|
+
memoryB: b,
|
|
690
|
+
type: 'entity_conflict',
|
|
691
|
+
description: `Potential conflict about ${entity}: "${a.summary}" vs "${b.summary}"`,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
return results.slice(0, limit);
|
|
699
|
+
}
|
|
700
|
+
// --------------------------------------------------------
|
|
701
|
+
// surface() — Proactive memory surfacing.
|
|
702
|
+
//
|
|
703
|
+
// The key insight from the manifesto: memories should be
|
|
704
|
+
// PUSHED when relevant, not just PULLED on demand.
|
|
705
|
+
//
|
|
706
|
+
// Unlike recall() which answers a question, surface() takes
|
|
707
|
+
// ambient context (what the agent is doing, what the user
|
|
708
|
+
// just said, what tool is running) and returns memories the
|
|
709
|
+
// agent didn't ask for but SHOULD know about right now.
|
|
710
|
+
//
|
|
711
|
+
// Returns empty array when nothing crosses the relevance
|
|
712
|
+
// threshold — silence is a valid response.
|
|
713
|
+
//
|
|
714
|
+
// Think of it like how a smell triggers a memory you weren't
|
|
715
|
+
// trying to recall.
|
|
716
|
+
// --------------------------------------------------------
|
|
717
|
+
async surface(input) {
|
|
718
|
+
const { context, activeEntities = [], activeTopics = [], seen = [], minSalience = 0.4, minHoursSinceAccess = 1, limit = 3, relevanceThreshold = 0.3, } = input;
|
|
719
|
+
const seenSet = new Set(seen);
|
|
720
|
+
const now = Date.now();
|
|
721
|
+
const minAccessAge = minHoursSinceAccess * 60 * 60 * 1000;
|
|
722
|
+
// Step 1: Run spreading activation to find contextually activated memories
|
|
723
|
+
// Use a wider net than normal recall — we want to find non-obvious connections
|
|
724
|
+
const candidates = new Map();
|
|
725
|
+
// Seed from active entities
|
|
726
|
+
for (const entity of activeEntities) {
|
|
727
|
+
const memories = this.store.getByEntity(entity, 30);
|
|
728
|
+
for (const mem of memories) {
|
|
729
|
+
this.addCandidate(candidates, mem, 0.6);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
// Seed from active topics
|
|
733
|
+
for (const topic of activeTopics) {
|
|
734
|
+
const memories = this.store.getByTopic(topic, 20);
|
|
735
|
+
for (const mem of memories) {
|
|
736
|
+
this.addCandidate(candidates, mem, 0.4);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
// Seed from context keywords
|
|
740
|
+
this.keywordSearch(context, candidates);
|
|
741
|
+
// Seed from semantic search if available
|
|
742
|
+
if (this.embedder && this.store.hasVectorSearch()) {
|
|
743
|
+
try {
|
|
744
|
+
const queryEmbedding = await this.embedder.embed(context);
|
|
745
|
+
const vectorResults = this.store.searchByVector(queryEmbedding, 20);
|
|
746
|
+
for (const vr of vectorResults) {
|
|
747
|
+
const mem = this.store.getMemoryDirect(vr.memoryId);
|
|
748
|
+
if (mem) {
|
|
749
|
+
const score = Math.max(0, 1 - vr.distance);
|
|
750
|
+
this.addCandidate(candidates, mem, score * 0.7);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
catch (_) { /* fallback already covered by keyword search */ }
|
|
755
|
+
}
|
|
756
|
+
// Run spreading activation with wider parameters
|
|
757
|
+
if (candidates.size > 0) {
|
|
758
|
+
this.spreadActivation(candidates, {
|
|
759
|
+
maxHops: 3, // Go deeper than normal recall
|
|
760
|
+
decay: 0.6, // Decay slower — we want distant surprises
|
|
761
|
+
minActivation: 0.08, // Lower threshold — cast a wider net
|
|
762
|
+
entityHops: true,
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
// Step 2: Filter for proactive-worthy memories
|
|
766
|
+
const results = [];
|
|
767
|
+
for (const [id, { memory, score }] of candidates) {
|
|
768
|
+
// Skip already-seen memories
|
|
769
|
+
if (seenSet.has(id))
|
|
770
|
+
continue;
|
|
771
|
+
// Skip low-salience memories (not important enough to proactively push)
|
|
772
|
+
if (memory.salience < minSalience)
|
|
773
|
+
continue;
|
|
774
|
+
// Skip recently accessed memories (don't repeat yourself)
|
|
775
|
+
const lastAccessed = new Date(memory.lastAccessedAt).getTime();
|
|
776
|
+
if (now - lastAccessed < minAccessAge)
|
|
777
|
+
continue;
|
|
778
|
+
// Skip archived/superseded
|
|
779
|
+
if (memory.status === 'archived' || memory.status === 'superseded')
|
|
780
|
+
continue;
|
|
781
|
+
// Must clear relevance threshold
|
|
782
|
+
if (score < relevanceThreshold)
|
|
783
|
+
continue;
|
|
784
|
+
// Determine WHY this is being surfaced
|
|
785
|
+
const reason = this.classifySurfaceReason(memory, context, activeEntities, activeTopics);
|
|
786
|
+
const activationPath = this.traceActivationPath(memory, activeEntities, activeTopics);
|
|
787
|
+
results.push({
|
|
788
|
+
memory,
|
|
789
|
+
reason,
|
|
790
|
+
relevance: Math.min(score, 1.0),
|
|
791
|
+
activationPath,
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
// Step 3: Rank by a composite score that favors:
|
|
795
|
+
// - High relevance (from spreading activation)
|
|
796
|
+
// - High salience (important memories)
|
|
797
|
+
// - Pending commitments (things that need attention)
|
|
798
|
+
// - Semantic type (facts and how-tos over raw episodes)
|
|
799
|
+
results.sort((a, b) => {
|
|
800
|
+
const scoreA = this.surfaceRankScore(a);
|
|
801
|
+
const scoreB = this.surfaceRankScore(b);
|
|
802
|
+
return scoreB - scoreA;
|
|
803
|
+
});
|
|
804
|
+
return results.slice(0, limit);
|
|
805
|
+
}
|
|
806
|
+
/** Classify why a memory is being proactively surfaced */
|
|
807
|
+
classifySurfaceReason(memory, context, activeEntities, activeTopics) {
|
|
808
|
+
// Pending commitment
|
|
809
|
+
if (memory.status === 'pending') {
|
|
810
|
+
return `Pending commitment: "${memory.summary}"`;
|
|
811
|
+
}
|
|
812
|
+
// Entity connection
|
|
813
|
+
const sharedEntities = memory.entities.filter(e => activeEntities.some(ae => ae.toLowerCase() === e.toLowerCase()));
|
|
814
|
+
if (sharedEntities.length > 0) {
|
|
815
|
+
return `Related to ${sharedEntities.join(', ')} in current context`;
|
|
816
|
+
}
|
|
817
|
+
// Topic overlap
|
|
818
|
+
const sharedTopics = memory.topics.filter(t => activeTopics.some(at => at.toLowerCase() === t.toLowerCase()));
|
|
819
|
+
if (sharedTopics.length > 0) {
|
|
820
|
+
return `Relevant topic: ${sharedTopics.join(', ')}`;
|
|
821
|
+
}
|
|
822
|
+
// Procedural (how-to that might help)
|
|
823
|
+
if (memory.type === 'procedural') {
|
|
824
|
+
return `Relevant procedure: "${memory.summary}"`;
|
|
825
|
+
}
|
|
826
|
+
// Semantic (fact that adds context)
|
|
827
|
+
if (memory.type === 'semantic') {
|
|
828
|
+
return `Background knowledge that may be relevant`;
|
|
829
|
+
}
|
|
830
|
+
return 'Activated through memory graph cascade';
|
|
831
|
+
}
|
|
832
|
+
/** Trace how a memory was activated (simplified path description) */
|
|
833
|
+
traceActivationPath(memory, activeEntities, activeTopics) {
|
|
834
|
+
const parts = [];
|
|
835
|
+
// Check direct entity match
|
|
836
|
+
const entityMatch = memory.entities.filter(e => activeEntities.some(ae => ae.toLowerCase() === e.toLowerCase()));
|
|
837
|
+
if (entityMatch.length > 0) {
|
|
838
|
+
parts.push(`entity:${entityMatch[0]}`);
|
|
839
|
+
}
|
|
840
|
+
// Check topic match
|
|
841
|
+
const topicMatch = memory.topics.filter(t => activeTopics.some(at => at.toLowerCase() === t.toLowerCase()));
|
|
842
|
+
if (topicMatch.length > 0) {
|
|
843
|
+
parts.push(`topic:${topicMatch[0]}`);
|
|
844
|
+
}
|
|
845
|
+
// Check graph edges
|
|
846
|
+
const edges = this.store.getEdgesBidirectional(memory.id);
|
|
847
|
+
if (edges.length > 0) {
|
|
848
|
+
const edgeTypes = [...new Set(edges.map(e => e.type))];
|
|
849
|
+
parts.push(`graph:${edgeTypes.join(',')}`);
|
|
850
|
+
}
|
|
851
|
+
if (parts.length === 0) {
|
|
852
|
+
// Must have been found via keyword/semantic similarity
|
|
853
|
+
parts.push('semantic_similarity');
|
|
854
|
+
}
|
|
855
|
+
return parts.join(' → ');
|
|
856
|
+
}
|
|
857
|
+
/** Composite ranking score for proactive surfacing */
|
|
858
|
+
surfaceRankScore(item) {
|
|
859
|
+
let score = item.relevance * 0.4; // Relevance from activation
|
|
860
|
+
score += item.memory.salience * 0.3; // Importance
|
|
861
|
+
score += item.memory.confidence * 0.1; // Trust
|
|
862
|
+
// Bonus for pending commitments (things that need attention)
|
|
863
|
+
if (item.memory.status === 'pending')
|
|
864
|
+
score += 0.15;
|
|
865
|
+
// Bonus for semantic/procedural (higher-value than raw episodes)
|
|
866
|
+
if (item.memory.type === 'semantic')
|
|
867
|
+
score += 0.05;
|
|
868
|
+
if (item.memory.type === 'procedural')
|
|
869
|
+
score += 0.08;
|
|
870
|
+
return score;
|
|
871
|
+
}
|
|
872
|
+
// --------------------------------------------------------
|
|
873
|
+
// stats() — Memory statistics
|
|
874
|
+
// --------------------------------------------------------
|
|
875
|
+
stats() {
|
|
876
|
+
return this.store.getStats();
|
|
877
|
+
}
|
|
878
|
+
// --------------------------------------------------------
|
|
879
|
+
// entities() — List all known entities
|
|
880
|
+
// --------------------------------------------------------
|
|
881
|
+
entities() {
|
|
882
|
+
return this.store.getAllEntities();
|
|
883
|
+
}
|
|
884
|
+
// --------------------------------------------------------
|
|
885
|
+
// export() — Full vault export
|
|
886
|
+
// --------------------------------------------------------
|
|
887
|
+
export() {
|
|
888
|
+
return this.store.exportAll();
|
|
889
|
+
}
|
|
890
|
+
// --------------------------------------------------------
|
|
891
|
+
// close() — Clean shutdown. Awaits all pending embeddings
|
|
892
|
+
// before closing the database to prevent data loss.
|
|
893
|
+
// --------------------------------------------------------
|
|
894
|
+
async close() {
|
|
895
|
+
if (this.pendingEmbeddings.size > 0) {
|
|
896
|
+
await Promise.allSettled([...this.pendingEmbeddings]);
|
|
897
|
+
}
|
|
898
|
+
this.store.close();
|
|
899
|
+
}
|
|
900
|
+
/** Flush all pending embedding computations without closing */
|
|
901
|
+
async flush() {
|
|
902
|
+
const count = this.pendingEmbeddings.size;
|
|
903
|
+
if (count > 0) {
|
|
904
|
+
await Promise.allSettled([...this.pendingEmbeddings]);
|
|
905
|
+
}
|
|
906
|
+
return count;
|
|
907
|
+
}
|
|
908
|
+
// --------------------------------------------------------
|
|
909
|
+
// extractWithLLM() — LLM-powered entity and topic extraction
|
|
910
|
+
// --------------------------------------------------------
|
|
911
|
+
async extractWithLLM(context) {
|
|
912
|
+
if (!this.config.llm) {
|
|
913
|
+
throw new Error('No LLM config available for extraction');
|
|
914
|
+
}
|
|
915
|
+
const prompt = `Extract entities and topics from this text for memory retrieval.
|
|
916
|
+
|
|
917
|
+
Text: "${context}"
|
|
918
|
+
|
|
919
|
+
Extract:
|
|
920
|
+
- ENTITIES: People, places, projects, companies, technologies, or specific things mentioned
|
|
921
|
+
- TOPICS: General themes, categories, or subjects (use simple, consistent terms)
|
|
922
|
+
|
|
923
|
+
Respond in this exact JSON format:
|
|
924
|
+
{
|
|
925
|
+
"entities": ["entity1", "entity2", ...],
|
|
926
|
+
"topics": ["topic1", "topic2", ...]
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
Keep entities specific and topics general. Limit to 10 entities and 8 topics max.`;
|
|
930
|
+
try {
|
|
931
|
+
const response = await this.callLLM('gemini-2.0-flash', prompt, this.config.llm);
|
|
932
|
+
const result = JSON.parse(response);
|
|
933
|
+
return {
|
|
934
|
+
entities: (result.entities || []).slice(0, 10),
|
|
935
|
+
topics: (result.topics || []).slice(0, 8),
|
|
936
|
+
};
|
|
937
|
+
}
|
|
938
|
+
catch (err) {
|
|
939
|
+
throw new Error(`LLM extraction failed: ${err}`);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
// --------------------------------------------------------
|
|
943
|
+
// Private: Rule-based consolidation (no LLM needed)
|
|
944
|
+
// --------------------------------------------------------
|
|
945
|
+
ruleBasedConsolidate(episodes) {
|
|
946
|
+
let semanticCreated = 0;
|
|
947
|
+
let entitiesDiscovered = 0;
|
|
948
|
+
let connectionsFormed = 0;
|
|
949
|
+
// 1. Find entity frequency patterns
|
|
950
|
+
const entityMentions = new Map();
|
|
951
|
+
for (const ep of episodes) {
|
|
952
|
+
for (const entity of ep.entities) {
|
|
953
|
+
entityMentions.set(entity, (entityMentions.get(entity) ?? 0) + 1);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
// Entities mentioned 3+ times get importance boost
|
|
957
|
+
for (const [entity, count] of entityMentions) {
|
|
958
|
+
if (count >= 3) {
|
|
959
|
+
const existing = this.store.getEntity(entity);
|
|
960
|
+
if (existing) {
|
|
961
|
+
// Boost importance
|
|
962
|
+
}
|
|
963
|
+
else {
|
|
964
|
+
this.store.upsertEntity(entity);
|
|
965
|
+
entitiesDiscovered++;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
// 2. Connect co-occurring memories
|
|
970
|
+
for (let i = 0; i < episodes.length; i++) {
|
|
971
|
+
for (let j = i + 1; j < episodes.length; j++) {
|
|
972
|
+
const shared = episodes[i].entities.filter(e => episodes[j].entities.includes(e));
|
|
973
|
+
if (shared.length > 0) {
|
|
974
|
+
this.store.createEdge(episodes[i].id, episodes[j].id, 'associated_with', Math.min(shared.length * 0.3, 1.0));
|
|
975
|
+
connectionsFormed++;
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
// 3. Create temporal sequence edges for consecutive episodes
|
|
980
|
+
for (let i = 0; i < episodes.length - 1; i++) {
|
|
981
|
+
this.store.createEdge(episodes[i].id, episodes[i + 1].id, 'temporal_next', 0.3);
|
|
982
|
+
connectionsFormed++;
|
|
983
|
+
}
|
|
984
|
+
return { semanticCreated, entitiesDiscovered, connectionsFormed };
|
|
985
|
+
}
|
|
986
|
+
// --------------------------------------------------------
|
|
987
|
+
// Private: LLM-powered consolidation
|
|
988
|
+
// --------------------------------------------------------
|
|
989
|
+
async llmConsolidate(episodes) {
|
|
990
|
+
if (episodes.length === 0) {
|
|
991
|
+
return { semanticCreated: 0, semanticUpdated: 0, entitiesDiscovered: 0, connectionsFormed: 0, contradictionsFound: 0 };
|
|
992
|
+
}
|
|
993
|
+
const llmConfig = this.config.llm;
|
|
994
|
+
const defaultModel = llmConfig.provider === 'gemini' ? 'gemini-2.0-flash'
|
|
995
|
+
: llmConfig.provider === 'openai' ? 'gpt-4o-mini'
|
|
996
|
+
: 'claude-3-5-haiku-20241022';
|
|
997
|
+
const model = llmConfig.model ?? defaultModel;
|
|
998
|
+
// Build the consolidation prompt
|
|
999
|
+
const episodeSummaries = episodes.map((e, i) => `[${i + 1}] (${e.createdAt}) ${e.content}`).join('\n');
|
|
1000
|
+
const existingSemanticMemories = this.store.getByType('semantic', 50);
|
|
1001
|
+
const existingContext = existingSemanticMemories.length > 0
|
|
1002
|
+
? `\n\nExisting knowledge:\n${existingSemanticMemories.map(m => `- ${m.content} (confidence: ${m.confidence})`).join('\n')}`
|
|
1003
|
+
: '';
|
|
1004
|
+
const prompt = `You are a memory consolidation engine. Analyze these recent episodic memories and extract structured knowledge.
|
|
1005
|
+
|
|
1006
|
+
Recent episodes:
|
|
1007
|
+
${episodeSummaries}
|
|
1008
|
+
${existingContext}
|
|
1009
|
+
|
|
1010
|
+
Extract:
|
|
1011
|
+
1. SEMANTIC MEMORIES: General facts, preferences, patterns that can be inferred from these episodes. Each should be a standalone statement of knowledge.
|
|
1012
|
+
2. ENTITIES: People, places, projects, or concepts mentioned. Include their type (person/place/project/concept) and any properties you can infer.
|
|
1013
|
+
3. CONTRADICTIONS: Any conflicts between these episodes or with existing knowledge.
|
|
1014
|
+
4. CONNECTIONS: Which episodes are related and how.
|
|
1015
|
+
|
|
1016
|
+
Respond in this exact JSON format:
|
|
1017
|
+
{
|
|
1018
|
+
"semantic_memories": [
|
|
1019
|
+
{"content": "...", "confidence": 0.0-1.0, "salience": 0.0-1.0, "entities": ["..."], "topics": ["..."]}
|
|
1020
|
+
],
|
|
1021
|
+
"entities": [
|
|
1022
|
+
{"name": "...", "type": "person|place|project|concept", "properties": {"key": "value"}}
|
|
1023
|
+
],
|
|
1024
|
+
"contradictions": [
|
|
1025
|
+
{"memory_a": "...", "memory_b": "...", "description": "..."}
|
|
1026
|
+
],
|
|
1027
|
+
"connections": [
|
|
1028
|
+
{"episode_a": 1, "episode_b": 2, "type": "supports|elaborates|causes|associated_with", "strength": 0.0-1.0}
|
|
1029
|
+
]
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
Be conservative with confidence scores. Only extract what's clearly supported by the episodes.`;
|
|
1033
|
+
try {
|
|
1034
|
+
const response = await this.callLLM(model, prompt, llmConfig);
|
|
1035
|
+
const result = JSON.parse(response);
|
|
1036
|
+
let semanticCreated = 0;
|
|
1037
|
+
let semanticUpdated = 0;
|
|
1038
|
+
let entitiesDiscovered = 0;
|
|
1039
|
+
let connectionsFormed = 0;
|
|
1040
|
+
const contradictionsFound = result.contradictions?.length ?? 0;
|
|
1041
|
+
// Create semantic memories (with dedup against existing semantics)
|
|
1042
|
+
for (const sem of result.semantic_memories ?? []) {
|
|
1043
|
+
// Check if a very similar semantic memory already exists
|
|
1044
|
+
// If so, update it instead of creating a duplicate
|
|
1045
|
+
let merged = false;
|
|
1046
|
+
if (this.embedder && this.store.hasVectorSearch()) {
|
|
1047
|
+
try {
|
|
1048
|
+
const embedding = await this.embedder.embed(sem.content);
|
|
1049
|
+
const similar = this.store.findSimilar(embedding, 0.15, 3); // slightly looser for consolidation
|
|
1050
|
+
for (const match of similar) {
|
|
1051
|
+
const existing = this.store.getMemoryDirect(match.memoryId);
|
|
1052
|
+
if (existing && existing.type === 'semantic' && existing.status === 'active') {
|
|
1053
|
+
// Update existing semantic memory: boost confidence & salience
|
|
1054
|
+
this.store.updateMemory(existing.id, {
|
|
1055
|
+
confidence: Math.min(1.0, Math.max(existing.confidence, sem.confidence ?? 0.7) + 0.05),
|
|
1056
|
+
salience: Math.min(1.0, Math.max(existing.salience, sem.salience ?? 0.5)),
|
|
1057
|
+
});
|
|
1058
|
+
semanticUpdated++;
|
|
1059
|
+
merged = true;
|
|
1060
|
+
break;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
catch {
|
|
1065
|
+
// Embedding failed — just create normally
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
if (!merged) {
|
|
1069
|
+
this.remember({
|
|
1070
|
+
content: sem.content,
|
|
1071
|
+
type: 'semantic',
|
|
1072
|
+
confidence: sem.confidence ?? 0.7,
|
|
1073
|
+
salience: sem.salience ?? 0.5,
|
|
1074
|
+
entities: sem.entities ?? [],
|
|
1075
|
+
topics: sem.topics ?? [],
|
|
1076
|
+
source: {
|
|
1077
|
+
type: 'consolidation',
|
|
1078
|
+
evidence: episodes.map(e => e.id),
|
|
1079
|
+
},
|
|
1080
|
+
});
|
|
1081
|
+
semanticCreated++;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
// Mark source episodes as superseded now that knowledge has been extracted
|
|
1085
|
+
for (const ep of episodes) {
|
|
1086
|
+
if (ep.status === 'active' && ep.type === 'episodic') {
|
|
1087
|
+
this.store.updateStatus(ep.id, 'superseded');
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
// Upsert entities
|
|
1091
|
+
for (const ent of result.entities ?? []) {
|
|
1092
|
+
this.store.upsertEntity(ent.name, ent.type);
|
|
1093
|
+
entitiesDiscovered++;
|
|
1094
|
+
}
|
|
1095
|
+
// Create connections
|
|
1096
|
+
for (const conn of result.connections ?? []) {
|
|
1097
|
+
const a = episodes[conn.episode_a - 1];
|
|
1098
|
+
const b = episodes[conn.episode_b - 1];
|
|
1099
|
+
if (a && b) {
|
|
1100
|
+
this.store.createEdge(a.id, b.id, conn.type, conn.strength ?? 0.5);
|
|
1101
|
+
connectionsFormed++;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
return { semanticCreated, semanticUpdated, entitiesDiscovered, connectionsFormed, contradictionsFound };
|
|
1105
|
+
}
|
|
1106
|
+
catch (err) {
|
|
1107
|
+
console.error('LLM consolidation failed:', err);
|
|
1108
|
+
// Fallback to rule-based
|
|
1109
|
+
const fallback = this.ruleBasedConsolidate(episodes);
|
|
1110
|
+
return { ...fallback, semanticUpdated: 0, contradictionsFound: 0 };
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
// --------------------------------------------------------
|
|
1114
|
+
// Private: LLM call
|
|
1115
|
+
// --------------------------------------------------------
|
|
1116
|
+
async callLLM(model, prompt, config) {
|
|
1117
|
+
if (config.provider === 'anthropic') {
|
|
1118
|
+
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
1119
|
+
method: 'POST',
|
|
1120
|
+
headers: {
|
|
1121
|
+
'Content-Type': 'application/json',
|
|
1122
|
+
'x-api-key': config.apiKey,
|
|
1123
|
+
'anthropic-version': '2023-06-01',
|
|
1124
|
+
},
|
|
1125
|
+
body: JSON.stringify({
|
|
1126
|
+
model,
|
|
1127
|
+
max_tokens: 4096,
|
|
1128
|
+
messages: [{ role: 'user', content: prompt }],
|
|
1129
|
+
}),
|
|
1130
|
+
});
|
|
1131
|
+
if (!response.ok) {
|
|
1132
|
+
throw new Error(`Anthropic API error: ${response.status} ${response.statusText}`);
|
|
1133
|
+
}
|
|
1134
|
+
const data = await response.json();
|
|
1135
|
+
const text = data.content?.find(c => c.type === 'text')?.text ?? '';
|
|
1136
|
+
// Extract JSON from response (handle markdown code blocks)
|
|
1137
|
+
const jsonMatch = text.match(/```json\s*([\s\S]*?)\s*```/) ?? text.match(/\{[\s\S]*\}/);
|
|
1138
|
+
return jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text;
|
|
1139
|
+
}
|
|
1140
|
+
if (config.provider === 'gemini') {
|
|
1141
|
+
const geminiModel = model.startsWith('gemini') ? model : 'gemini-2.0-flash';
|
|
1142
|
+
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${config.apiKey}`, {
|
|
1143
|
+
method: 'POST',
|
|
1144
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1145
|
+
body: JSON.stringify({
|
|
1146
|
+
contents: [{ parts: [{ text: prompt }] }],
|
|
1147
|
+
generationConfig: {
|
|
1148
|
+
responseMimeType: 'application/json',
|
|
1149
|
+
maxOutputTokens: 4096,
|
|
1150
|
+
},
|
|
1151
|
+
}),
|
|
1152
|
+
});
|
|
1153
|
+
if (!response.ok) {
|
|
1154
|
+
const err = await response.text();
|
|
1155
|
+
throw new Error(`Gemini API error: ${response.status} ${err}`);
|
|
1156
|
+
}
|
|
1157
|
+
const data = await response.json();
|
|
1158
|
+
const text = data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
|
|
1159
|
+
// Gemini with responseMimeType=application/json should return clean JSON
|
|
1160
|
+
const jsonMatch = text.match(/```json\s*([\s\S]*?)\s*```/) ?? text.match(/\{[\s\S]*\}/);
|
|
1161
|
+
return jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text;
|
|
1162
|
+
}
|
|
1163
|
+
if (config.provider === 'openai') {
|
|
1164
|
+
const response = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
1165
|
+
method: 'POST',
|
|
1166
|
+
headers: {
|
|
1167
|
+
'Content-Type': 'application/json',
|
|
1168
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
1169
|
+
},
|
|
1170
|
+
body: JSON.stringify({
|
|
1171
|
+
model,
|
|
1172
|
+
messages: [{ role: 'user', content: prompt }],
|
|
1173
|
+
response_format: { type: 'json_object' },
|
|
1174
|
+
}),
|
|
1175
|
+
});
|
|
1176
|
+
if (!response.ok) {
|
|
1177
|
+
throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
|
|
1178
|
+
}
|
|
1179
|
+
const data = await response.json();
|
|
1180
|
+
return data.choices[0]?.message?.content ?? '';
|
|
1181
|
+
}
|
|
1182
|
+
throw new Error(`Unsupported LLM provider: ${config.provider}`);
|
|
1183
|
+
}
|
|
1184
|
+
// --------------------------------------------------------
|
|
1185
|
+
// Private: Keyword search fallback
|
|
1186
|
+
// --------------------------------------------------------
|
|
1187
|
+
keywordSearch(context, candidates, baseScore = 0.3) {
|
|
1188
|
+
const keywords = this.extractKeywords(context);
|
|
1189
|
+
for (const keyword of keywords.slice(0, 5)) {
|
|
1190
|
+
const memories = this.store.search(keyword, 10);
|
|
1191
|
+
for (const mem of memories) {
|
|
1192
|
+
this.addCandidate(candidates, mem, baseScore);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
// --------------------------------------------------------
|
|
1197
|
+
// Private: Keyword extraction (simple, no LLM needed)
|
|
1198
|
+
// --------------------------------------------------------
|
|
1199
|
+
extractKeywords(text) {
|
|
1200
|
+
const stopWords = new Set([
|
|
1201
|
+
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
1202
|
+
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
|
|
1203
|
+
'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
|
|
1204
|
+
'on', 'with', 'at', 'by', 'from', 'as', 'into', 'about', 'between',
|
|
1205
|
+
'through', 'during', 'before', 'after', 'above', 'below', 'and', 'but',
|
|
1206
|
+
'or', 'nor', 'not', 'so', 'yet', 'both', 'either', 'neither', 'each',
|
|
1207
|
+
'every', 'all', 'any', 'few', 'more', 'most', 'other', 'some', 'such',
|
|
1208
|
+
'no', 'only', 'own', 'same', 'than', 'too', 'very', 'just', 'because',
|
|
1209
|
+
'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'it', 'they',
|
|
1210
|
+
'them', 'his', 'her', 'its', 'their', 'what', 'which', 'who', 'whom',
|
|
1211
|
+
'this', 'that', 'these', 'those', 'am', 'if', 'then', 'else', 'when',
|
|
1212
|
+
'up', 'out', 'off', 'over', 'under', 'again', 'further', 'once',
|
|
1213
|
+
]);
|
|
1214
|
+
return text
|
|
1215
|
+
.toLowerCase()
|
|
1216
|
+
.replace(/[^\w\s]/g, '')
|
|
1217
|
+
.split(/\s+/)
|
|
1218
|
+
.filter(word => word.length > 2 && !stopWords.has(word))
|
|
1219
|
+
.slice(0, 10);
|
|
1220
|
+
}
|
|
1221
|
+
// --------------------------------------------------------
|
|
1222
|
+
// Private: Candidate scoring helper
|
|
1223
|
+
// --------------------------------------------------------
|
|
1224
|
+
addCandidate(candidates, memory, score) {
|
|
1225
|
+
const existing = candidates.get(memory.id);
|
|
1226
|
+
if (existing) {
|
|
1227
|
+
existing.score = Math.min(existing.score + score, 1.0); // Boost for multiple retrieval paths
|
|
1228
|
+
}
|
|
1229
|
+
else {
|
|
1230
|
+
candidates.set(memory.id, { memory, score });
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
//# sourceMappingURL=vault.js.map
|