engram-sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CONTRIBUTING.md +65 -0
  2. package/Dockerfile +21 -0
  3. package/EVAL-FRAMEWORK.md +70 -0
  4. package/EVAL.md +127 -0
  5. package/LICENSE +17 -0
  6. package/README.md +309 -0
  7. package/ROADMAP.md +113 -0
  8. package/deploy/fly.toml +26 -0
  9. package/dist/auto-ingest.d.ts +3 -0
  10. package/dist/auto-ingest.d.ts.map +1 -0
  11. package/dist/auto-ingest.js +334 -0
  12. package/dist/auto-ingest.js.map +1 -0
  13. package/dist/brief.d.ts +45 -0
  14. package/dist/brief.d.ts.map +1 -0
  15. package/dist/brief.js +183 -0
  16. package/dist/brief.js.map +1 -0
  17. package/dist/claude-watcher.d.ts +3 -0
  18. package/dist/claude-watcher.d.ts.map +1 -0
  19. package/dist/claude-watcher.js +385 -0
  20. package/dist/claude-watcher.js.map +1 -0
  21. package/dist/cli.d.ts +3 -0
  22. package/dist/cli.d.ts.map +1 -0
  23. package/dist/cli.js +764 -0
  24. package/dist/cli.js.map +1 -0
  25. package/dist/embeddings.d.ts +42 -0
  26. package/dist/embeddings.d.ts.map +1 -0
  27. package/dist/embeddings.js +145 -0
  28. package/dist/embeddings.js.map +1 -0
  29. package/dist/eval.d.ts +2 -0
  30. package/dist/eval.d.ts.map +1 -0
  31. package/dist/eval.js +281 -0
  32. package/dist/eval.js.map +1 -0
  33. package/dist/extract.d.ts +11 -0
  34. package/dist/extract.d.ts.map +1 -0
  35. package/dist/extract.js +139 -0
  36. package/dist/extract.js.map +1 -0
  37. package/dist/hosted.d.ts +3 -0
  38. package/dist/hosted.d.ts.map +1 -0
  39. package/dist/hosted.js +144 -0
  40. package/dist/hosted.js.map +1 -0
  41. package/dist/index.d.ts +11 -0
  42. package/dist/index.d.ts.map +1 -0
  43. package/dist/index.js +7 -0
  44. package/dist/index.js.map +1 -0
  45. package/dist/ingest.d.ts +28 -0
  46. package/dist/ingest.d.ts.map +1 -0
  47. package/dist/ingest.js +192 -0
  48. package/dist/ingest.js.map +1 -0
  49. package/dist/mcp.d.ts +3 -0
  50. package/dist/mcp.d.ts.map +1 -0
  51. package/dist/mcp.js +349 -0
  52. package/dist/mcp.js.map +1 -0
  53. package/dist/server.d.ts +17 -0
  54. package/dist/server.d.ts.map +1 -0
  55. package/dist/server.js +515 -0
  56. package/dist/server.js.map +1 -0
  57. package/dist/store.d.ts +87 -0
  58. package/dist/store.d.ts.map +1 -0
  59. package/dist/store.js +548 -0
  60. package/dist/store.js.map +1 -0
  61. package/dist/types.d.ts +204 -0
  62. package/dist/types.d.ts.map +1 -0
  63. package/dist/types.js +77 -0
  64. package/dist/types.js.map +1 -0
  65. package/dist/vault.d.ts +116 -0
  66. package/dist/vault.d.ts.map +1 -0
  67. package/dist/vault.js +1234 -0
  68. package/dist/vault.js.map +1 -0
  69. package/package.json +61 -0
package/dist/vault.js ADDED
@@ -0,0 +1,1234 @@
1
+ import path from 'path';
2
+ import { MemoryStore } from './store.js';
3
+ import { RememberInputSchema, RecallInputSchema } from './types.js';
4
+ import { extract } from './extract.js';
5
+ // ============================================================
6
+ // Vault — The public API for Engram
7
+ // ============================================================
8
+ export class Vault {
9
+ store;
10
+ config;
11
+ embedder = null;
12
+ /** Track all in-flight embedding computations so close() can await them */
13
+ pendingEmbeddings = new Set();
14
+ constructor(config, embedder) {
15
+ this.config = config;
16
+ this.embedder = embedder ?? null;
17
+ const dbPath = config.dbPath ?? path.resolve(`engram-${config.owner}.db`);
18
+ this.store = new MemoryStore(dbPath, embedder?.dimensions());
19
+ }
20
+ // --------------------------------------------------------
21
+ // remember() — Store a new memory
22
+ // --------------------------------------------------------
23
+ remember(input) {
24
+ // Accept a plain string for convenience
25
+ const parsed = typeof input === 'string'
26
+ ? RememberInputSchema.parse({ content: input })
27
+ : RememberInputSchema.parse(input);
28
+ // Auto-extract entities and topics if not provided
29
+ if (parsed.entities.length === 0 && parsed.topics.length === 0) {
30
+ const extracted = extract(parsed.content);
31
+ if (parsed.entities.length === 0)
32
+ parsed.entities = extracted.entities;
33
+ if (parsed.topics.length === 0)
34
+ parsed.topics = extracted.topics;
35
+ // Only use suggested salience if user didn't set one (default is 0.5)
36
+ if (parsed.salience === 0.5)
37
+ parsed.salience = extracted.suggestedSalience;
38
+ }
39
+ // Auto-set source metadata from vault config
40
+ if (!parsed.source) {
41
+ parsed.source = { type: 'conversation' };
42
+ }
43
+ if (this.config.agentId && !parsed.source.agentId) {
44
+ parsed.source.agentId = this.config.agentId;
45
+ }
46
+ if (this.config.sessionId && !parsed.source.sessionId) {
47
+ parsed.source.sessionId = this.config.sessionId;
48
+ }
49
+ const memory = this.store.createMemory(parsed);
50
+ // Queue embedding computation (non-blocking but tracked)
51
+ // Also checks for near-duplicates after embedding is computed.
52
+ if (this.embedder) {
53
+ const p = this.computeAndStoreEmbedding(memory.id, memory.content)
54
+ .then(() => {
55
+ // Dedup check: if a very similar memory already exists, merge instead of keeping both
56
+ this.dedup(memory);
57
+ })
58
+ .catch(err => {
59
+ console.warn(`Failed to compute embedding for ${memory.id}:`, err);
60
+ })
61
+ .finally(() => {
62
+ this.pendingEmbeddings.delete(p);
63
+ });
64
+ this.pendingEmbeddings.add(p);
65
+ }
66
+ return memory;
67
+ }
68
+ /**
69
+ * Dedup: after storing a memory and its embedding, check if a near-identical
70
+ * memory already exists. If so, keep the better one (higher salience/confidence,
71
+ * or newer if semantic) and supersede the other.
72
+ *
73
+ * Threshold: cosine distance <= 0.08 (similarity >= 0.92) = near-duplicate.
74
+ * Only dedup within the same type (don't merge episodic into semantic).
75
+ */
76
+ dedup(memory) {
77
+ // Don't dedup consolidation outputs — they're intentional
78
+ if (memory.source?.type === 'consolidation')
79
+ return;
80
+ try {
81
+ const similar = this.store.findSimilar(this.store.getEmbedding(memory.id) ?? [], 0.08, // cosine distance threshold — very tight, ~92% similarity
82
+ 5);
83
+ for (const match of similar) {
84
+ if (match.memoryId === memory.id)
85
+ continue; // skip self
86
+ const existing = this.store.getMemoryDirect(match.memoryId);
87
+ if (!existing)
88
+ continue;
89
+ if (existing.status !== 'active')
90
+ continue;
91
+ if (existing.type !== memory.type)
92
+ continue; // only dedup same type
93
+ // We have a near-duplicate. Keep the one with higher salience,
94
+ // or if equal, keep the newer one (more up-to-date).
95
+ const keepNew = memory.salience >= existing.salience;
96
+ const supersededId = keepNew ? existing.id : memory.id;
97
+ const keptId = keepNew ? memory.id : existing.id;
98
+ this.store.updateStatus(supersededId, 'superseded');
99
+ // Create a supersedes edge so the graph tracks the lineage
100
+ this.store.createEdge(keptId, supersededId, 'supersedes', 0.8);
101
+ // Merge: boost the kept memory's salience slightly from the duplicate
102
+ const kept = this.store.getMemoryDirect(keptId);
103
+ if (kept && kept.salience < 1.0) {
104
+ this.store.updateMemory(keptId, {
105
+ salience: Math.min(1.0, kept.salience + 0.05),
106
+ });
107
+ }
108
+ break; // Only process one duplicate match
109
+ }
110
+ }
111
+ catch {
112
+ // Dedup is best-effort; don't break remember() if it fails
113
+ }
114
+ }
115
+ /** Compute embedding and store it — can be awaited if needed */
116
+ async computeAndStoreEmbedding(memoryId, content) {
117
+ if (!this.embedder)
118
+ return;
119
+ const embedding = await this.embedder.embed(content);
120
+ this.store.storeEmbedding(memoryId, embedding);
121
+ }
122
+ /** Batch compute embeddings for all memories missing them */
123
+ async backfillEmbeddings() {
124
+ if (!this.embedder)
125
+ return 0;
126
+ const allMemories = this.store.exportAll().memories;
127
+ let count = 0;
128
+ // Process in batches of 50
129
+ for (let i = 0; i < allMemories.length; i += 50) {
130
+ const batch = allMemories.slice(i, i + 50);
131
+ const texts = batch.map(m => m.content);
132
+ const embeddings = await this.embedder.embedBatch(texts);
133
+ for (let j = 0; j < batch.length; j++) {
134
+ this.store.storeEmbedding(batch[j].id, embeddings[j]);
135
+ count++;
136
+ }
137
+ }
138
+ return count;
139
+ }
140
+ // --------------------------------------------------------
141
+ // recall() — Retrieve relevant memories for a context
142
+ // --------------------------------------------------------
143
+ async recall(input) {
144
+ const parsed = typeof input === 'string'
145
+ ? RecallInputSchema.parse({ context: input })
146
+ : RecallInputSchema.parse(input);
147
+ const candidates = new Map();
148
+ // ── Phase 0: Auto-extract entities and topics from query ──
149
+ // If the caller didn't provide explicit entities/topics,
150
+ // extract them from the context string so entity/topic
151
+ // retrieval actually fires. Try LLM extraction first if available,
152
+ // then fall back to rule-based extraction.
153
+ if ((!parsed.entities || parsed.entities.length === 0) ||
154
+ (!parsed.topics || parsed.topics.length === 0)) {
155
+ // Try LLM extraction if vault has LLM config
156
+ if (this.config.llm && (!parsed.entities || parsed.entities.length === 0) && (!parsed.topics || parsed.topics.length === 0)) {
157
+ try {
158
+ const llmExtracted = await this.extractWithLLM(parsed.context);
159
+ if (!parsed.entities || parsed.entities.length === 0) {
160
+ parsed.entities = llmExtracted.entities;
161
+ }
162
+ if (!parsed.topics || parsed.topics.length === 0) {
163
+ parsed.topics = llmExtracted.topics;
164
+ }
165
+ }
166
+ catch (err) {
167
+ // LLM extraction failed — fall back to rule-based
168
+ console.warn('LLM extraction failed, falling back to rule-based:', err);
169
+ const extracted = extract(parsed.context);
170
+ if (!parsed.entities || parsed.entities.length === 0) {
171
+ parsed.entities = extracted.entities;
172
+ }
173
+ if (!parsed.topics || parsed.topics.length === 0) {
174
+ parsed.topics = extracted.topics;
175
+ }
176
+ }
177
+ }
178
+ else {
179
+ // No LLM config or entities/topics already provided — use rule-based
180
+ const extracted = extract(parsed.context);
181
+ if (!parsed.entities || parsed.entities.length === 0) {
182
+ parsed.entities = extracted.entities;
183
+ }
184
+ if (!parsed.topics || parsed.topics.length === 0) {
185
+ parsed.topics = extracted.topics;
186
+ }
187
+ }
188
+ }
189
+ // ── Phase 0b: Aggregation detection ──
190
+ const aggregationPatterns = [
191
+ /\ball\b.*\b(commitments?|promises?|pending|outstanding)\b/i,
192
+ /\b(pending|outstanding|unfulfilled)\b.*\b(commitments?|promises?|tasks?)\b/i,
193
+ /\b(corrected|updated|changed|revised)\b/i,
194
+ /\bkey\s+(metrics?|numbers?|stats?|statistics?|KPIs?)\b/i,
195
+ /\bevery\b|\blist\s+(of|all)\b|\bcomplete\s+list\b/i,
196
+ ];
197
+ const isAggregation = aggregationPatterns.some(p => p.test(parsed.context));
198
+ if (isAggregation) {
199
+ const aggTopics = parsed.topics ?? [];
200
+ for (const topic of aggTopics) {
201
+ const topicMemories = this.store.getByTopic(topic, 50);
202
+ for (const mem of topicMemories) {
203
+ this.addCandidate(candidates, mem, 0.3);
204
+ }
205
+ }
206
+ if (/commitment|pending|promise|outstanding|unfulfilled/i.test(parsed.context)) {
207
+ const pendingMemories = this.store.getByStatus('pending', 50);
208
+ for (const mem of pendingMemories) {
209
+ this.addCandidate(candidates, mem, 0.4);
210
+ }
211
+ const fulfilledMemories = this.store.getByStatus('fulfilled', 50);
212
+ for (const mem of fulfilledMemories) {
213
+ this.addCandidate(candidates, mem, 0.3);
214
+ }
215
+ }
216
+ if (/correct|update|change|revis|wrong/i.test(parsed.context)) {
217
+ const supersededMemories = this.store.getByStatus('superseded', 30);
218
+ for (const mem of supersededMemories) {
219
+ this.addCandidate(candidates, mem, 0.35);
220
+ }
221
+ const correctionMemories = this.store.getByTopic('correction', 30);
222
+ for (const mem of correctionMemories) {
223
+ this.addCandidate(candidates, mem, 0.35);
224
+ }
225
+ }
226
+ if (/metric|number|stat|KPI|measure/i.test(parsed.context)) {
227
+ const metricsMemories = this.store.getByTopic('metrics', 50);
228
+ for (const mem of metricsMemories) {
229
+ this.addCandidate(candidates, mem, 0.35);
230
+ }
231
+ }
232
+ parsed.limit = Math.max(parsed.limit, 30);
233
+ }
234
+ // ── Phase 1: Direct retrieval (seed memories) ──────────
235
+ // ── Phase 1 Strategy ──
236
+ // Vector search is the primary retrieval signal — it finds
237
+ // what's semantically relevant to the query. Entity/topic
238
+ // matching acts as a secondary boost, not a primary retriever,
239
+ // because common entities (e.g. "Thomas" in 100+ memories)
240
+ // flood the candidate pool with noise if scored too high.
241
+ // 1. Semantic search via embeddings (PRIMARY — highest signal)
242
+ if (this.embedder && this.store.hasVectorSearch()) {
243
+ try {
244
+ const queryEmbedding = await this.embedder.embed(parsed.context);
245
+ const vectorResults = this.store.searchByVector(queryEmbedding, 50);
246
+ for (const vr of vectorResults) {
247
+ const mem = this.store.getMemoryDirect(vr.memoryId);
248
+ if (mem) {
249
+ // Use cosine similarity (1 - distance) as primary score
250
+ const similarity = Math.max(0, 1 - vr.distance);
251
+ this.addCandidate(candidates, mem, similarity);
252
+ }
253
+ }
254
+ }
255
+ catch (err) {
256
+ // Vector search failed — keyword search becomes primary
257
+ this.keywordSearch(parsed.context, candidates, 0.4);
258
+ }
259
+ }
260
+ else {
261
+ // No embeddings available — keyword is primary
262
+ this.keywordSearch(parsed.context, candidates, 0.4);
263
+ }
264
+ // 1b. Keyword search (ALWAYS runs as supplementary signal)
265
+ // Catches exact term matches that embeddings might miss —
266
+ // e.g. "competitors" in a query matching "competitors" in content.
267
+ this.keywordSearch(parsed.context, candidates, 0.2);
268
+ // 2. Entity-based retrieval (SECONDARY — boost, not flood)
269
+ // Pull more candidates for entity matches but weight by type:
270
+ // semantic memories are higher signal (consolidated knowledge)
271
+ // than raw episodic entries.
272
+ if (parsed.entities && parsed.entities.length > 0) {
273
+ for (const entity of parsed.entities) {
274
+ const memories = this.store.getByEntity(entity, 20);
275
+ for (const mem of memories) {
276
+ // Scale entity base score: fewer results = higher confidence each is relevant
277
+ const baseScore = memories.length <= 5 ? 0.25 : memories.length <= 15 ? 0.15 : 0.1;
278
+ // Semantic memories from entity match get a bonus — they're distilled facts
279
+ const typeBonus = mem.type === 'semantic' ? 0.1 : 0;
280
+ this.addCandidate(candidates, mem, baseScore + typeBonus);
281
+ }
282
+ }
283
+ }
284
+ // 3. Topic-based retrieval (SECONDARY)
285
+ if (parsed.topics && parsed.topics.length > 0) {
286
+ for (const topic of parsed.topics) {
287
+ const memories = this.store.getByTopic(topic, 10);
288
+ const topicScore = memories.length <= 3 ? 0.2 : 0.08;
289
+ for (const mem of memories) {
290
+ this.addCandidate(candidates, mem, topicScore);
291
+ }
292
+ }
293
+ }
294
+ // 4. Recent memories (light recency signal)
295
+ const recent = this.store.getRecent(5);
296
+ for (const mem of recent) {
297
+ this.addCandidate(candidates, mem, 0.05);
298
+ }
299
+ // ── Phase 2: Spreading activation ──────────────────────
300
+ // Take the seeds from Phase 1 and let activation cascade
301
+ // through the memory graph. This is what makes recall feel
302
+ // like memory instead of search.
303
+ if (parsed.spread && candidates.size > 0) {
304
+ this.spreadActivation(candidates, {
305
+ maxHops: parsed.spreadHops,
306
+ decay: parsed.spreadDecay,
307
+ minActivation: parsed.spreadMinActivation,
308
+ entityHops: parsed.spreadEntityHops,
309
+ });
310
+ }
311
+ // ── Phase 3: Filter, score, rank ───────────────────────
312
+ // 5. Filter out superseded/archived memories first
313
+ const showSuperseded = isAggregation && /correct|update|change|revis/i.test(parsed.context);
314
+ let results = [...candidates.values()].filter(r => r.memory.status !== 'archived' &&
315
+ (r.memory.status !== 'superseded' || showSuperseded));
316
+ // Type filter
317
+ if (parsed.types && parsed.types.length > 0) {
318
+ results = results.filter(r => parsed.types.includes(r.memory.type));
319
+ }
320
+ // 6. Apply minimum thresholds
321
+ results = results.filter(r => r.memory.salience >= parsed.minSalience &&
322
+ r.memory.confidence >= parsed.minConfidence);
323
+ // 7. Temporal focus
324
+ if (parsed.temporalFocus === 'recent') {
325
+ const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
326
+ results = results.filter(r => r.memory.createdAt >= oneWeekAgo);
327
+ }
328
+ // 8. Score with salience, stability, and type weighting
329
+ // Semantic memories with high stability are the "core knowledge" —
330
+ // they should outrank noisy episodic results when the vector
331
+ // similarity scores are close. Without this, basic factual queries
332
+ // like "What is Thomas's job?" get buried under episodic noise.
333
+ for (const r of results) {
334
+ const cappedStability = Math.min(r.memory.stability, 3.0);
335
+ // Base weight from salience and stability
336
+ const salienceBoost = r.memory.salience * 0.25;
337
+ const stabilityBoost = cappedStability * 0.1;
338
+ // Type bonus: consolidated semantic memories are higher-signal
339
+ // than raw episodic entries for factual queries
340
+ const typeBonus = r.memory.type === 'semantic' ? 0.25 : 0;
341
+ // Confidence bonus: high-confidence memories are more reliable
342
+ const confidenceBonus = r.memory.confidence * 0.05;
343
+ // Superseded/archived penalty: shouldn't appear in results
344
+ const statusPenalty = (r.memory.status === 'superseded' || r.memory.status === 'archived') ? 0.5 : 0;
345
+ r.score = r.score * (0.5 + salienceBoost + stabilityBoost + typeBonus + confidenceBonus) - statusPenalty;
346
+ }
347
+ // 9. Sort by score and return top N
348
+ results.sort((a, b) => b.score - a.score);
349
+ // Mark accessed (only the returned results, not traversal noise)
350
+ const topResults = results.slice(0, parsed.limit);
351
+ for (const r of topResults) {
352
+ this.store.getMemory(r.memory.id); // Triggers access count + stability update
353
+ }
354
+ return topResults.map(r => r.memory);
355
+ }
356
+ // --------------------------------------------------------
357
+ // Spreading Activation — The cascade that makes recall
358
+ // feel like memory instead of search.
359
+ //
360
+ // Algorithm:
361
+ // 1. Seeds come in with initial activation scores from Phase 1
362
+ // 2. For each hop:
363
+ // a. Collect all edges from currently active memories
364
+ // b. For each neighbor: activation = parent_activation × edge_strength × decay
365
+ // c. Also spread via shared entities (implicit edges)
366
+ // d. Add/boost neighbor in candidate pool
367
+ // 3. Stop when activation falls below threshold or max hops reached
368
+ //
369
+ // This is why querying "Thomas" can surface his marathon training
370
+ // schedule even if you only asked about his work preferences.
371
+ // --------------------------------------------------------
372
+ spreadActivation(candidates, opts) {
373
+ // Current frontier: memory IDs and their activation level
374
+ let frontier = new Map();
375
+ // Initialize frontier from current candidates
376
+ for (const [id, { score }] of candidates) {
377
+ frontier.set(id, score);
378
+ }
379
+ const visited = new Set(frontier.keys());
380
+ for (let hop = 0; hop < opts.maxHops; hop++) {
381
+ const nextFrontier = new Map();
382
+ const frontierIds = [...frontier.keys()];
383
+ if (frontierIds.length === 0)
384
+ break;
385
+ // ── Edge-based spreading ──
386
+ const edges = this.store.getEdgesForMemories(frontierIds);
387
+ for (const edge of edges) {
388
+ const parentId = frontier.has(edge.sourceId) ? edge.sourceId : edge.targetId;
389
+ const neighborId = edge.sourceId === parentId ? edge.targetId : edge.sourceId;
390
+ const parentActivation = frontier.get(parentId) ?? 0;
391
+ // Activation = parent × edge_strength × decay × edge_type_weight
392
+ const typeWeight = this.edgeTypeWeight(edge.type);
393
+ const activation = parentActivation * edge.strength * opts.decay * typeWeight;
394
+ if (activation < opts.minActivation)
395
+ continue;
396
+ // Accumulate activation (multiple paths can reinforce)
397
+ const existing = nextFrontier.get(neighborId) ?? 0;
398
+ nextFrontier.set(neighborId, Math.min(existing + activation, 1.0));
399
+ }
400
+ // ── Entity-based spreading (implicit edges) ──
401
+ // Memories that share entities are implicitly connected.
402
+ // This is crucial when the explicit graph is sparse.
403
+ if (opts.entityHops) {
404
+ for (const id of frontierIds) {
405
+ const parentActivation = frontier.get(id) ?? 0;
406
+ const coEntities = this.store.getCoEntityMemories(id, 10);
407
+ for (const { memory: neighbor, sharedEntities } of coEntities) {
408
+ if (visited.has(neighbor.id))
409
+ continue;
410
+ // More shared entities = stronger implicit connection
411
+ const implicitStrength = Math.min(sharedEntities.length * 0.3, 0.9);
412
+ const activation = parentActivation * implicitStrength * opts.decay;
413
+ if (activation < opts.minActivation)
414
+ continue;
415
+ const existing = nextFrontier.get(neighbor.id) ?? 0;
416
+ nextFrontier.set(neighbor.id, Math.min(existing + activation, 1.0));
417
+ }
418
+ }
419
+ }
420
+ // Load activated memories and add to candidates
421
+ const newIds = [...nextFrontier.keys()].filter(id => !visited.has(id));
422
+ if (newIds.length === 0 && [...nextFrontier.keys()].every(id => visited.has(id)))
423
+ break;
424
+ const newMemories = this.store.getMemoriesDirect(newIds);
425
+ const memoryMap = new Map(newMemories.map(m => [m.id, m]));
426
+ for (const [id, activation] of nextFrontier) {
427
+ const memory = memoryMap.get(id) ?? candidates.get(id)?.memory;
428
+ if (!memory)
429
+ continue;
430
+ // Tag that this came from spreading (for debugging/eval)
431
+ // Use a reduced weight — spread results shouldn't dominate direct hits
432
+ const spreadWeight = 0.6;
433
+ this.addCandidate(candidates, memory, activation * spreadWeight);
434
+ visited.add(id);
435
+ }
436
+ // Next hop starts from newly activated memories
437
+ frontier = new Map();
438
+ for (const [id, activation] of nextFrontier) {
439
+ if (activation >= opts.minActivation) {
440
+ frontier.set(id, activation);
441
+ }
442
+ }
443
+ }
444
+ }
445
+ // --------------------------------------------------------
446
+ // Edge type weights — how strongly different relationship
447
+ // types propagate activation.
448
+ // --------------------------------------------------------
449
+ edgeTypeWeight(type) {
450
+ switch (type) {
451
+ case 'supports': return 0.9; // Strong: supporting evidence propagates well
452
+ case 'elaborates': return 0.85; // Strong: detail enriches context
453
+ case 'causes': return 0.8; // Causal chains are highly relevant
454
+ case 'caused_by': return 0.8;
455
+ case 'part_of': return 0.75; // Part-whole relationships matter
456
+ case 'instance_of': return 0.7; // Specific→general is useful
457
+ case 'supersedes': return 0.6; // Updated info still connects
458
+ case 'associated_with': return 0.5; // Weak but valid
459
+ case 'temporal_next': return 0.4; // Temporal sequence is loose
460
+ case 'derived_from': return 0.7; // Consolidation lineage
461
+ case 'contradicts': return 0.3; // Contradictions are relevant but shouldn't dominate
462
+ default: return 0.5;
463
+ }
464
+ }
465
+ // --------------------------------------------------------
466
+ // forget() — Explicitly remove or decay a memory
467
+ // --------------------------------------------------------
468
+ forget(id, hard = false) {
469
+ if (hard) {
470
+ this.store.deleteMemory(id);
471
+ }
472
+ else {
473
+ // Soft forget: drastically reduce stability
474
+ this.store.updateMemory(id, { salience: 0 });
475
+ }
476
+ }
477
+ // --------------------------------------------------------
478
+ // connect() — Create a relationship between memories
479
+ // --------------------------------------------------------
480
+ connect(sourceId, targetId, type, strength = 0.5) {
481
+ return this.store.createEdge(sourceId, targetId, type, strength);
482
+ }
483
+ // --------------------------------------------------------
484
+ // neighbors() — Get related memories via graph traversal
485
+ // --------------------------------------------------------
486
+ neighbors(memoryId, depth = 1) {
487
+ return this.store.getNeighbors(memoryId, depth);
488
+ }
489
+ // --------------------------------------------------------
490
+ // consolidate() — The magic: turn episodes into knowledge
491
+ // --------------------------------------------------------
492
+ async consolidate() {
493
+ const startedAt = new Date().toISOString();
494
+ // Get recent unconsolidated episodes
495
+ const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
496
+ const episodes = this.store.getEpisodicSince(oneDayAgo);
497
+ let semanticCreated = 0;
498
+ let semanticUpdated = 0;
499
+ let entitiesDiscovered = 0;
500
+ let connectionsFormed = 0;
501
+ let contradictionsFound = 0;
502
+ if (this.config.llm) {
503
+ // LLM-powered consolidation
504
+ const result = await this.llmConsolidate(episodes);
505
+ semanticCreated = result.semanticCreated;
506
+ semanticUpdated = result.semanticUpdated;
507
+ entitiesDiscovered = result.entitiesDiscovered;
508
+ connectionsFormed = result.connectionsFormed;
509
+ contradictionsFound = result.contradictionsFound;
510
+ }
511
+ else {
512
+ // Rule-based consolidation (no LLM required)
513
+ const result = this.ruleBasedConsolidate(episodes);
514
+ semanticCreated = result.semanticCreated;
515
+ entitiesDiscovered = result.entitiesDiscovered;
516
+ connectionsFormed = result.connectionsFormed;
517
+ }
518
+ // Apply decay
519
+ const decayed = this.store.applyDecay(this.config.decay?.halfLifeHours ?? 168);
520
+ // Archive deeply decayed memories
521
+ const archived = this.store.getDecayedMemories(this.config.decay?.archiveThreshold ?? 0.05);
522
+ for (const mem of archived) {
523
+ this.store.deleteMemory(mem.id); // TODO: move to cold storage instead of deleting
524
+ }
525
+ const report = {
526
+ startedAt,
527
+ completedAt: new Date().toISOString(),
528
+ episodesProcessed: episodes.length,
529
+ semanticMemoriesCreated: semanticCreated,
530
+ semanticMemoriesUpdated: semanticUpdated,
531
+ entitiesDiscovered,
532
+ connectionsFormed,
533
+ contradictionsFound,
534
+ memoriesDecayed: decayed,
535
+ memoriesArchived: archived.length,
536
+ };
537
+ // Store the consolidation report as a memory itself
538
+ this.remember({
539
+ content: `Consolidation completed: processed ${episodes.length} episodes, created ${semanticCreated} semantic memories, discovered ${entitiesDiscovered} entities, formed ${connectionsFormed} connections, decayed ${decayed} memories.`,
540
+ type: 'procedural',
541
+ topics: ['meta', 'consolidation'],
542
+ salience: 0.3,
543
+ source: { type: 'consolidation' },
544
+ });
545
+ return report;
546
+ }
547
+ // --------------------------------------------------------
548
+ // briefing() — Structured context summary for session start.
549
+ // This is the MEMORY.md replacement: instead of reading a
550
+ // flat file, an agent calls POST /v1/briefing and gets a
551
+ // curated knowledge snapshot.
552
+ // --------------------------------------------------------
553
+ async briefing(context = '', limit = 20) {
554
+ // 1. High-salience semantic memories (key facts)
555
+ const allSemantic = this.store.getByType('semantic', 100);
556
+ const keyFacts = allSemantic
557
+ .filter(m => m.salience >= 0.4 && m.status === 'active')
558
+ .sort((a, b) => b.salience - a.salience)
559
+ .slice(0, limit)
560
+ .map(m => ({ content: m.content, salience: m.salience, entities: m.entities }));
561
+ // 2. Active commitments (pending status)
562
+ const allMemories = this.store.exportAll().memories;
563
+ const activeCommitments = allMemories
564
+ .filter(m => m.status === 'pending')
565
+ .sort((a, b) => b.salience - a.salience)
566
+ .slice(0, 10)
567
+ .map(m => ({ content: m.content, status: m.status, entities: m.entities }));
568
+ // 3. Recent activity (last 24h episodes)
569
+ const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
570
+ const recentActivity = this.store.getEpisodicSince(oneDayAgo, 10)
571
+ .map(m => ({ content: m.content, when: m.createdAt }));
572
+ // 4. Top entities
573
+ const topEntities = this.entities()
574
+ .slice(0, 10)
575
+ .map(e => ({ name: e.name, type: e.type, memoryCount: e.memoryCount }));
576
+ // 5. Contradictions
577
+ const contradictions = this.contradictions(5)
578
+ .map(c => ({ a: c.memoryA.content, b: c.memoryB.content }));
579
+ // 6. If context is provided, do a spreading-activation recall and weave in results
580
+ let contextualMemories = [];
581
+ if (context.trim()) {
582
+ const recalled = await this.recall({ context, limit: 5, spread: true });
583
+ contextualMemories = recalled.map(m => m.content);
584
+ }
585
+ // 7. Build summary
586
+ const stats = this.stats();
587
+ const summaryParts = [];
588
+ summaryParts.push(`Vault: ${stats.total} memories (${stats.semantic} semantic, ${stats.episodic} episodic, ${stats.procedural} procedural), ${stats.entities} entities.`);
589
+ if (activeCommitments.length > 0) {
590
+ summaryParts.push(`${activeCommitments.length} pending commitment(s).`);
591
+ }
592
+ if (contradictions.length > 0) {
593
+ summaryParts.push(`${contradictions.length} unresolved contradiction(s).`);
594
+ }
595
+ if (contextualMemories.length > 0) {
596
+ summaryParts.push(`Context-relevant: ${contextualMemories.join(' | ')}`);
597
+ }
598
+ return {
599
+ summary: summaryParts.join(' '),
600
+ keyFacts,
601
+ activeCommitments,
602
+ recentActivity,
603
+ topEntities,
604
+ contradictions,
605
+ stats,
606
+ };
607
+ }
608
+ // --------------------------------------------------------
609
+ // contradictions() — Find unresolved conflicts in the graph.
610
+ // No competitor has this. It's a real differentiator.
611
+ //
612
+ // Checks:
613
+ // 1. Explicit 'contradicts' edges in the graph
614
+ // 2. Status conflicts (superseded memories with active successors)
615
+ // 3. Entity-scoped content conflicts (LLM-powered if available)
616
+ // --------------------------------------------------------
617
+ contradictions(limit = 50) {
618
+ const results = [];
619
+ // 1. Explicit contradiction edges
620
+ const allExport = this.store.exportAll();
621
+ const memoryMap = new Map(allExport.memories.map(m => [m.id, m]));
622
+ for (const edge of allExport.edges) {
623
+ if (edge.type === 'contradicts') {
624
+ const a = memoryMap.get(edge.sourceId);
625
+ const b = memoryMap.get(edge.targetId);
626
+ if (a && b) {
627
+ results.push({
628
+ memoryA: a,
629
+ memoryB: b,
630
+ type: 'explicit_edge',
631
+ description: `Explicit contradiction (edge strength: ${edge.strength.toFixed(2)})`,
632
+ });
633
+ }
634
+ }
635
+ if (edge.type === 'supersedes') {
636
+ const newer = memoryMap.get(edge.sourceId);
637
+ const older = memoryMap.get(edge.targetId);
638
+ if (newer && older && older.status === 'active') {
639
+ results.push({
640
+ memoryA: newer,
641
+ memoryB: older,
642
+ type: 'superseded_conflict',
643
+ description: `"${newer.summary}" supersedes "${older.summary}" but older is still marked active`,
644
+ });
645
+ }
646
+ }
647
+ }
648
+ // 2. Find potential entity-scoped conflicts
649
+ // Group semantic memories by entity, look for opposing claims
650
+ const entityMemories = new Map();
651
+ for (const mem of allExport.memories) {
652
+ if (mem.type !== 'semantic' || mem.status !== 'active')
653
+ continue;
654
+ for (const entity of mem.entities) {
655
+ const list = entityMemories.get(entity) ?? [];
656
+ list.push(mem);
657
+ entityMemories.set(entity, list);
658
+ }
659
+ }
660
+ // Simple heuristic: if two semantic memories about the same entity
661
+ // have conflicting signals (negation words, opposite qualifiers)
662
+ const negationPatterns = [
663
+ /\bnot\b/i, /\bnever\b/i, /\bno longer\b/i, /\bstopped\b/i,
664
+ /\bwon't\b/i, /\bdoesn't\b/i, /\bisn't\b/i, /\bwasn't\b/i,
665
+ /\bhates?\b/i, /\bdislikes?\b/i, /\bavoids?\b/i,
666
+ ];
667
+ const affirmationPatterns = [
668
+ /\balways\b/i, /\bloves?\b/i, /\bprefers?\b/i, /\bfavorite\b/i,
669
+ /\bregularly\b/i, /\bevery\b/i, /\benjoying\b/i,
670
+ ];
671
+ for (const [entity, mems] of entityMemories) {
672
+ if (mems.length < 2)
673
+ continue;
674
+ for (let i = 0; i < mems.length && results.length < limit; i++) {
675
+ for (let j = i + 1; j < mems.length && results.length < limit; j++) {
676
+ const a = mems[i];
677
+ const b = mems[j];
678
+ const aHasNeg = negationPatterns.some(p => p.test(a.content));
679
+ const bHasAff = affirmationPatterns.some(p => p.test(b.content));
680
+ const aHasAff = affirmationPatterns.some(p => p.test(a.content));
681
+ const bHasNeg = negationPatterns.some(p => p.test(b.content));
682
+ if ((aHasNeg && bHasAff) || (aHasAff && bHasNeg)) {
683
+ // Check they're about a similar topic (share >1 entity or topic)
684
+ const sharedEntities = a.entities.filter(e => b.entities.includes(e));
685
+ const sharedTopics = a.topics.filter(t => b.topics.includes(t));
686
+ if (sharedEntities.length + sharedTopics.length >= 1) {
687
+ results.push({
688
+ memoryA: a,
689
+ memoryB: b,
690
+ type: 'entity_conflict',
691
+ description: `Potential conflict about ${entity}: "${a.summary}" vs "${b.summary}"`,
692
+ });
693
+ }
694
+ }
695
+ }
696
+ }
697
+ }
698
+ return results.slice(0, limit);
699
+ }
700
+ // --------------------------------------------------------
701
+ // surface() — Proactive memory surfacing.
702
+ //
703
+ // The key insight from the manifesto: memories should be
704
+ // PUSHED when relevant, not just PULLED on demand.
705
+ //
706
+ // Unlike recall() which answers a question, surface() takes
707
+ // ambient context (what the agent is doing, what the user
708
+ // just said, what tool is running) and returns memories the
709
+ // agent didn't ask for but SHOULD know about right now.
710
+ //
711
+ // Returns empty array when nothing crosses the relevance
712
+ // threshold — silence is a valid response.
713
+ //
714
+ // Think of it like how a smell triggers a memory you weren't
715
+ // trying to recall.
716
+ // --------------------------------------------------------
717
+ async surface(input) {
718
+ const { context, activeEntities = [], activeTopics = [], seen = [], minSalience = 0.4, minHoursSinceAccess = 1, limit = 3, relevanceThreshold = 0.3, } = input;
719
+ const seenSet = new Set(seen);
720
+ const now = Date.now();
721
+ const minAccessAge = minHoursSinceAccess * 60 * 60 * 1000;
722
+ // Step 1: Run spreading activation to find contextually activated memories
723
+ // Use a wider net than normal recall — we want to find non-obvious connections
724
+ const candidates = new Map();
725
+ // Seed from active entities
726
+ for (const entity of activeEntities) {
727
+ const memories = this.store.getByEntity(entity, 30);
728
+ for (const mem of memories) {
729
+ this.addCandidate(candidates, mem, 0.6);
730
+ }
731
+ }
732
+ // Seed from active topics
733
+ for (const topic of activeTopics) {
734
+ const memories = this.store.getByTopic(topic, 20);
735
+ for (const mem of memories) {
736
+ this.addCandidate(candidates, mem, 0.4);
737
+ }
738
+ }
739
+ // Seed from context keywords
740
+ this.keywordSearch(context, candidates);
741
+ // Seed from semantic search if available
742
+ if (this.embedder && this.store.hasVectorSearch()) {
743
+ try {
744
+ const queryEmbedding = await this.embedder.embed(context);
745
+ const vectorResults = this.store.searchByVector(queryEmbedding, 20);
746
+ for (const vr of vectorResults) {
747
+ const mem = this.store.getMemoryDirect(vr.memoryId);
748
+ if (mem) {
749
+ const score = Math.max(0, 1 - vr.distance);
750
+ this.addCandidate(candidates, mem, score * 0.7);
751
+ }
752
+ }
753
+ }
754
+ catch (_) { /* fallback already covered by keyword search */ }
755
+ }
756
+ // Run spreading activation with wider parameters
757
+ if (candidates.size > 0) {
758
+ this.spreadActivation(candidates, {
759
+ maxHops: 3, // Go deeper than normal recall
760
+ decay: 0.6, // Decay slower — we want distant surprises
761
+ minActivation: 0.08, // Lower threshold — cast a wider net
762
+ entityHops: true,
763
+ });
764
+ }
765
+ // Step 2: Filter for proactive-worthy memories
766
+ const results = [];
767
+ for (const [id, { memory, score }] of candidates) {
768
+ // Skip already-seen memories
769
+ if (seenSet.has(id))
770
+ continue;
771
+ // Skip low-salience memories (not important enough to proactively push)
772
+ if (memory.salience < minSalience)
773
+ continue;
774
+ // Skip recently accessed memories (don't repeat yourself)
775
+ const lastAccessed = new Date(memory.lastAccessedAt).getTime();
776
+ if (now - lastAccessed < minAccessAge)
777
+ continue;
778
+ // Skip archived/superseded
779
+ if (memory.status === 'archived' || memory.status === 'superseded')
780
+ continue;
781
+ // Must clear relevance threshold
782
+ if (score < relevanceThreshold)
783
+ continue;
784
+ // Determine WHY this is being surfaced
785
+ const reason = this.classifySurfaceReason(memory, context, activeEntities, activeTopics);
786
+ const activationPath = this.traceActivationPath(memory, activeEntities, activeTopics);
787
+ results.push({
788
+ memory,
789
+ reason,
790
+ relevance: Math.min(score, 1.0),
791
+ activationPath,
792
+ });
793
+ }
794
+ // Step 3: Rank by a composite score that favors:
795
+ // - High relevance (from spreading activation)
796
+ // - High salience (important memories)
797
+ // - Pending commitments (things that need attention)
798
+ // - Semantic type (facts and how-tos over raw episodes)
799
+ results.sort((a, b) => {
800
+ const scoreA = this.surfaceRankScore(a);
801
+ const scoreB = this.surfaceRankScore(b);
802
+ return scoreB - scoreA;
803
+ });
804
+ return results.slice(0, limit);
805
+ }
806
+ /** Classify why a memory is being proactively surfaced */
807
+ classifySurfaceReason(memory, context, activeEntities, activeTopics) {
808
+ // Pending commitment
809
+ if (memory.status === 'pending') {
810
+ return `Pending commitment: "${memory.summary}"`;
811
+ }
812
+ // Entity connection
813
+ const sharedEntities = memory.entities.filter(e => activeEntities.some(ae => ae.toLowerCase() === e.toLowerCase()));
814
+ if (sharedEntities.length > 0) {
815
+ return `Related to ${sharedEntities.join(', ')} in current context`;
816
+ }
817
+ // Topic overlap
818
+ const sharedTopics = memory.topics.filter(t => activeTopics.some(at => at.toLowerCase() === t.toLowerCase()));
819
+ if (sharedTopics.length > 0) {
820
+ return `Relevant topic: ${sharedTopics.join(', ')}`;
821
+ }
822
+ // Procedural (how-to that might help)
823
+ if (memory.type === 'procedural') {
824
+ return `Relevant procedure: "${memory.summary}"`;
825
+ }
826
+ // Semantic (fact that adds context)
827
+ if (memory.type === 'semantic') {
828
+ return `Background knowledge that may be relevant`;
829
+ }
830
+ return 'Activated through memory graph cascade';
831
+ }
832
+ /** Trace how a memory was activated (simplified path description) */
833
+ traceActivationPath(memory, activeEntities, activeTopics) {
834
+ const parts = [];
835
+ // Check direct entity match
836
+ const entityMatch = memory.entities.filter(e => activeEntities.some(ae => ae.toLowerCase() === e.toLowerCase()));
837
+ if (entityMatch.length > 0) {
838
+ parts.push(`entity:${entityMatch[0]}`);
839
+ }
840
+ // Check topic match
841
+ const topicMatch = memory.topics.filter(t => activeTopics.some(at => at.toLowerCase() === t.toLowerCase()));
842
+ if (topicMatch.length > 0) {
843
+ parts.push(`topic:${topicMatch[0]}`);
844
+ }
845
+ // Check graph edges
846
+ const edges = this.store.getEdgesBidirectional(memory.id);
847
+ if (edges.length > 0) {
848
+ const edgeTypes = [...new Set(edges.map(e => e.type))];
849
+ parts.push(`graph:${edgeTypes.join(',')}`);
850
+ }
851
+ if (parts.length === 0) {
852
+ // Must have been found via keyword/semantic similarity
853
+ parts.push('semantic_similarity');
854
+ }
855
+ return parts.join(' → ');
856
+ }
857
+ /** Composite ranking score for proactive surfacing */
858
+ surfaceRankScore(item) {
859
+ let score = item.relevance * 0.4; // Relevance from activation
860
+ score += item.memory.salience * 0.3; // Importance
861
+ score += item.memory.confidence * 0.1; // Trust
862
+ // Bonus for pending commitments (things that need attention)
863
+ if (item.memory.status === 'pending')
864
+ score += 0.15;
865
+ // Bonus for semantic/procedural (higher-value than raw episodes)
866
+ if (item.memory.type === 'semantic')
867
+ score += 0.05;
868
+ if (item.memory.type === 'procedural')
869
+ score += 0.08;
870
+ return score;
871
+ }
872
+ // --------------------------------------------------------
873
+ // stats() — Memory statistics
874
+ // --------------------------------------------------------
875
+ stats() {
876
+ return this.store.getStats();
877
+ }
878
+ // --------------------------------------------------------
879
+ // entities() — List all known entities
880
+ // --------------------------------------------------------
881
+ entities() {
882
+ return this.store.getAllEntities();
883
+ }
884
+ // --------------------------------------------------------
885
+ // export() — Full vault export
886
+ // --------------------------------------------------------
887
+ export() {
888
+ return this.store.exportAll();
889
+ }
890
+ // --------------------------------------------------------
891
+ // close() — Clean shutdown. Awaits all pending embeddings
892
+ // before closing the database to prevent data loss.
893
+ // --------------------------------------------------------
894
+ async close() {
895
+ if (this.pendingEmbeddings.size > 0) {
896
+ await Promise.allSettled([...this.pendingEmbeddings]);
897
+ }
898
+ this.store.close();
899
+ }
900
+ /** Flush all pending embedding computations without closing */
901
+ async flush() {
902
+ const count = this.pendingEmbeddings.size;
903
+ if (count > 0) {
904
+ await Promise.allSettled([...this.pendingEmbeddings]);
905
+ }
906
+ return count;
907
+ }
908
+ // --------------------------------------------------------
909
+ // extractWithLLM() — LLM-powered entity and topic extraction
910
+ // --------------------------------------------------------
911
+ async extractWithLLM(context) {
912
+ if (!this.config.llm) {
913
+ throw new Error('No LLM config available for extraction');
914
+ }
915
+ const prompt = `Extract entities and topics from this text for memory retrieval.
916
+
917
+ Text: "${context}"
918
+
919
+ Extract:
920
+ - ENTITIES: People, places, projects, companies, technologies, or specific things mentioned
921
+ - TOPICS: General themes, categories, or subjects (use simple, consistent terms)
922
+
923
+ Respond in this exact JSON format:
924
+ {
925
+ "entities": ["entity1", "entity2", ...],
926
+ "topics": ["topic1", "topic2", ...]
927
+ }
928
+
929
+ Keep entities specific and topics general. Limit to 10 entities and 8 topics max.`;
930
+ try {
931
+ const response = await this.callLLM('gemini-2.0-flash', prompt, this.config.llm);
932
+ const result = JSON.parse(response);
933
+ return {
934
+ entities: (result.entities || []).slice(0, 10),
935
+ topics: (result.topics || []).slice(0, 8),
936
+ };
937
+ }
938
+ catch (err) {
939
+ throw new Error(`LLM extraction failed: ${err}`);
940
+ }
941
+ }
942
+ // --------------------------------------------------------
943
+ // Private: Rule-based consolidation (no LLM needed)
944
+ // --------------------------------------------------------
945
+ ruleBasedConsolidate(episodes) {
946
+ let semanticCreated = 0;
947
+ let entitiesDiscovered = 0;
948
+ let connectionsFormed = 0;
949
+ // 1. Find entity frequency patterns
950
+ const entityMentions = new Map();
951
+ for (const ep of episodes) {
952
+ for (const entity of ep.entities) {
953
+ entityMentions.set(entity, (entityMentions.get(entity) ?? 0) + 1);
954
+ }
955
+ }
956
+ // Entities mentioned 3+ times get importance boost
957
+ for (const [entity, count] of entityMentions) {
958
+ if (count >= 3) {
959
+ const existing = this.store.getEntity(entity);
960
+ if (existing) {
961
+ // Boost importance
962
+ }
963
+ else {
964
+ this.store.upsertEntity(entity);
965
+ entitiesDiscovered++;
966
+ }
967
+ }
968
+ }
969
+ // 2. Connect co-occurring memories
970
+ for (let i = 0; i < episodes.length; i++) {
971
+ for (let j = i + 1; j < episodes.length; j++) {
972
+ const shared = episodes[i].entities.filter(e => episodes[j].entities.includes(e));
973
+ if (shared.length > 0) {
974
+ this.store.createEdge(episodes[i].id, episodes[j].id, 'associated_with', Math.min(shared.length * 0.3, 1.0));
975
+ connectionsFormed++;
976
+ }
977
+ }
978
+ }
979
+ // 3. Create temporal sequence edges for consecutive episodes
980
+ for (let i = 0; i < episodes.length - 1; i++) {
981
+ this.store.createEdge(episodes[i].id, episodes[i + 1].id, 'temporal_next', 0.3);
982
+ connectionsFormed++;
983
+ }
984
+ return { semanticCreated, entitiesDiscovered, connectionsFormed };
985
+ }
986
+ // --------------------------------------------------------
987
+ // Private: LLM-powered consolidation
988
+ // --------------------------------------------------------
989
+ async llmConsolidate(episodes) {
990
+ if (episodes.length === 0) {
991
+ return { semanticCreated: 0, semanticUpdated: 0, entitiesDiscovered: 0, connectionsFormed: 0, contradictionsFound: 0 };
992
+ }
993
+ const llmConfig = this.config.llm;
994
+ const defaultModel = llmConfig.provider === 'gemini' ? 'gemini-2.0-flash'
995
+ : llmConfig.provider === 'openai' ? 'gpt-4o-mini'
996
+ : 'claude-3-5-haiku-20241022';
997
+ const model = llmConfig.model ?? defaultModel;
998
+ // Build the consolidation prompt
999
+ const episodeSummaries = episodes.map((e, i) => `[${i + 1}] (${e.createdAt}) ${e.content}`).join('\n');
1000
+ const existingSemanticMemories = this.store.getByType('semantic', 50);
1001
+ const existingContext = existingSemanticMemories.length > 0
1002
+ ? `\n\nExisting knowledge:\n${existingSemanticMemories.map(m => `- ${m.content} (confidence: ${m.confidence})`).join('\n')}`
1003
+ : '';
1004
+ const prompt = `You are a memory consolidation engine. Analyze these recent episodic memories and extract structured knowledge.
1005
+
1006
+ Recent episodes:
1007
+ ${episodeSummaries}
1008
+ ${existingContext}
1009
+
1010
+ Extract:
1011
+ 1. SEMANTIC MEMORIES: General facts, preferences, patterns that can be inferred from these episodes. Each should be a standalone statement of knowledge.
1012
+ 2. ENTITIES: People, places, projects, or concepts mentioned. Include their type (person/place/project/concept) and any properties you can infer.
1013
+ 3. CONTRADICTIONS: Any conflicts between these episodes or with existing knowledge.
1014
+ 4. CONNECTIONS: Which episodes are related and how.
1015
+
1016
+ Respond in this exact JSON format:
1017
+ {
1018
+ "semantic_memories": [
1019
+ {"content": "...", "confidence": 0.0-1.0, "salience": 0.0-1.0, "entities": ["..."], "topics": ["..."]}
1020
+ ],
1021
+ "entities": [
1022
+ {"name": "...", "type": "person|place|project|concept", "properties": {"key": "value"}}
1023
+ ],
1024
+ "contradictions": [
1025
+ {"memory_a": "...", "memory_b": "...", "description": "..."}
1026
+ ],
1027
+ "connections": [
1028
+ {"episode_a": 1, "episode_b": 2, "type": "supports|elaborates|causes|associated_with", "strength": 0.0-1.0}
1029
+ ]
1030
+ }
1031
+
1032
+ Be conservative with confidence scores. Only extract what's clearly supported by the episodes.`;
1033
+ try {
1034
+ const response = await this.callLLM(model, prompt, llmConfig);
1035
+ const result = JSON.parse(response);
1036
+ let semanticCreated = 0;
1037
+ let semanticUpdated = 0;
1038
+ let entitiesDiscovered = 0;
1039
+ let connectionsFormed = 0;
1040
+ const contradictionsFound = result.contradictions?.length ?? 0;
1041
+ // Create semantic memories (with dedup against existing semantics)
1042
+ for (const sem of result.semantic_memories ?? []) {
1043
+ // Check if a very similar semantic memory already exists
1044
+ // If so, update it instead of creating a duplicate
1045
+ let merged = false;
1046
+ if (this.embedder && this.store.hasVectorSearch()) {
1047
+ try {
1048
+ const embedding = await this.embedder.embed(sem.content);
1049
+ const similar = this.store.findSimilar(embedding, 0.15, 3); // slightly looser for consolidation
1050
+ for (const match of similar) {
1051
+ const existing = this.store.getMemoryDirect(match.memoryId);
1052
+ if (existing && existing.type === 'semantic' && existing.status === 'active') {
1053
+ // Update existing semantic memory: boost confidence & salience
1054
+ this.store.updateMemory(existing.id, {
1055
+ confidence: Math.min(1.0, Math.max(existing.confidence, sem.confidence ?? 0.7) + 0.05),
1056
+ salience: Math.min(1.0, Math.max(existing.salience, sem.salience ?? 0.5)),
1057
+ });
1058
+ semanticUpdated++;
1059
+ merged = true;
1060
+ break;
1061
+ }
1062
+ }
1063
+ }
1064
+ catch {
1065
+ // Embedding failed — just create normally
1066
+ }
1067
+ }
1068
+ if (!merged) {
1069
+ this.remember({
1070
+ content: sem.content,
1071
+ type: 'semantic',
1072
+ confidence: sem.confidence ?? 0.7,
1073
+ salience: sem.salience ?? 0.5,
1074
+ entities: sem.entities ?? [],
1075
+ topics: sem.topics ?? [],
1076
+ source: {
1077
+ type: 'consolidation',
1078
+ evidence: episodes.map(e => e.id),
1079
+ },
1080
+ });
1081
+ semanticCreated++;
1082
+ }
1083
+ }
1084
+ // Mark source episodes as superseded now that knowledge has been extracted
1085
+ for (const ep of episodes) {
1086
+ if (ep.status === 'active' && ep.type === 'episodic') {
1087
+ this.store.updateStatus(ep.id, 'superseded');
1088
+ }
1089
+ }
1090
+ // Upsert entities
1091
+ for (const ent of result.entities ?? []) {
1092
+ this.store.upsertEntity(ent.name, ent.type);
1093
+ entitiesDiscovered++;
1094
+ }
1095
+ // Create connections
1096
+ for (const conn of result.connections ?? []) {
1097
+ const a = episodes[conn.episode_a - 1];
1098
+ const b = episodes[conn.episode_b - 1];
1099
+ if (a && b) {
1100
+ this.store.createEdge(a.id, b.id, conn.type, conn.strength ?? 0.5);
1101
+ connectionsFormed++;
1102
+ }
1103
+ }
1104
+ return { semanticCreated, semanticUpdated, entitiesDiscovered, connectionsFormed, contradictionsFound };
1105
+ }
1106
+ catch (err) {
1107
+ console.error('LLM consolidation failed:', err);
1108
+ // Fallback to rule-based
1109
+ const fallback = this.ruleBasedConsolidate(episodes);
1110
+ return { ...fallback, semanticUpdated: 0, contradictionsFound: 0 };
1111
+ }
1112
+ }
1113
+ // --------------------------------------------------------
1114
+ // Private: LLM call
1115
+ // --------------------------------------------------------
1116
+ async callLLM(model, prompt, config) {
1117
+ if (config.provider === 'anthropic') {
1118
+ const response = await fetch('https://api.anthropic.com/v1/messages', {
1119
+ method: 'POST',
1120
+ headers: {
1121
+ 'Content-Type': 'application/json',
1122
+ 'x-api-key': config.apiKey,
1123
+ 'anthropic-version': '2023-06-01',
1124
+ },
1125
+ body: JSON.stringify({
1126
+ model,
1127
+ max_tokens: 4096,
1128
+ messages: [{ role: 'user', content: prompt }],
1129
+ }),
1130
+ });
1131
+ if (!response.ok) {
1132
+ throw new Error(`Anthropic API error: ${response.status} ${response.statusText}`);
1133
+ }
1134
+ const data = await response.json();
1135
+ const text = data.content?.find(c => c.type === 'text')?.text ?? '';
1136
+ // Extract JSON from response (handle markdown code blocks)
1137
+ const jsonMatch = text.match(/```json\s*([\s\S]*?)\s*```/) ?? text.match(/\{[\s\S]*\}/);
1138
+ return jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text;
1139
+ }
1140
+ if (config.provider === 'gemini') {
1141
+ const geminiModel = model.startsWith('gemini') ? model : 'gemini-2.0-flash';
1142
+ const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${config.apiKey}`, {
1143
+ method: 'POST',
1144
+ headers: { 'Content-Type': 'application/json' },
1145
+ body: JSON.stringify({
1146
+ contents: [{ parts: [{ text: prompt }] }],
1147
+ generationConfig: {
1148
+ responseMimeType: 'application/json',
1149
+ maxOutputTokens: 4096,
1150
+ },
1151
+ }),
1152
+ });
1153
+ if (!response.ok) {
1154
+ const err = await response.text();
1155
+ throw new Error(`Gemini API error: ${response.status} ${err}`);
1156
+ }
1157
+ const data = await response.json();
1158
+ const text = data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
1159
+ // Gemini with responseMimeType=application/json should return clean JSON
1160
+ const jsonMatch = text.match(/```json\s*([\s\S]*?)\s*```/) ?? text.match(/\{[\s\S]*\}/);
1161
+ return jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text;
1162
+ }
1163
+ if (config.provider === 'openai') {
1164
+ const response = await fetch('https://api.openai.com/v1/chat/completions', {
1165
+ method: 'POST',
1166
+ headers: {
1167
+ 'Content-Type': 'application/json',
1168
+ 'Authorization': `Bearer ${config.apiKey}`,
1169
+ },
1170
+ body: JSON.stringify({
1171
+ model,
1172
+ messages: [{ role: 'user', content: prompt }],
1173
+ response_format: { type: 'json_object' },
1174
+ }),
1175
+ });
1176
+ if (!response.ok) {
1177
+ throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
1178
+ }
1179
+ const data = await response.json();
1180
+ return data.choices[0]?.message?.content ?? '';
1181
+ }
1182
+ throw new Error(`Unsupported LLM provider: ${config.provider}`);
1183
+ }
1184
+ // --------------------------------------------------------
1185
+ // Private: Keyword search fallback
1186
+ // --------------------------------------------------------
1187
+ keywordSearch(context, candidates, baseScore = 0.3) {
1188
+ const keywords = this.extractKeywords(context);
1189
+ for (const keyword of keywords.slice(0, 5)) {
1190
+ const memories = this.store.search(keyword, 10);
1191
+ for (const mem of memories) {
1192
+ this.addCandidate(candidates, mem, baseScore);
1193
+ }
1194
+ }
1195
+ }
1196
+ // --------------------------------------------------------
1197
+ // Private: Keyword extraction (simple, no LLM needed)
1198
+ // --------------------------------------------------------
1199
+ extractKeywords(text) {
1200
+ const stopWords = new Set([
1201
+ 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
1202
+ 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
1203
+ 'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
1204
+ 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'about', 'between',
1205
+ 'through', 'during', 'before', 'after', 'above', 'below', 'and', 'but',
1206
+ 'or', 'nor', 'not', 'so', 'yet', 'both', 'either', 'neither', 'each',
1207
+ 'every', 'all', 'any', 'few', 'more', 'most', 'other', 'some', 'such',
1208
+ 'no', 'only', 'own', 'same', 'than', 'too', 'very', 'just', 'because',
1209
+ 'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'it', 'they',
1210
+ 'them', 'his', 'her', 'its', 'their', 'what', 'which', 'who', 'whom',
1211
+ 'this', 'that', 'these', 'those', 'am', 'if', 'then', 'else', 'when',
1212
+ 'up', 'out', 'off', 'over', 'under', 'again', 'further', 'once',
1213
+ ]);
1214
+ return text
1215
+ .toLowerCase()
1216
+ .replace(/[^\w\s]/g, '')
1217
+ .split(/\s+/)
1218
+ .filter(word => word.length > 2 && !stopWords.has(word))
1219
+ .slice(0, 10);
1220
+ }
1221
+ // --------------------------------------------------------
1222
+ // Private: Candidate scoring helper
1223
+ // --------------------------------------------------------
1224
+ addCandidate(candidates, memory, score) {
1225
+ const existing = candidates.get(memory.id);
1226
+ if (existing) {
1227
+ existing.score = Math.min(existing.score + score, 1.0); // Boost for multiple retrieval paths
1228
+ }
1229
+ else {
1230
+ candidates.set(memory.id, { memory, score });
1231
+ }
1232
+ }
1233
+ }
1234
+ //# sourceMappingURL=vault.js.map