@monoes/memory 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -7
- package/dist/controller-registry.d.ts +1 -1
- package/dist/controller-registry.d.ts.map +1 -1
- package/dist/controller-registry.js +3 -3
- package/dist/controller-registry.js.map +1 -1
- package/dist/database-provider.d.ts +1 -2
- package/dist/database-provider.d.ts.map +1 -1
- package/dist/database-provider.js +0 -25
- package/dist/database-provider.js.map +1 -1
- package/dist/index.d.ts +0 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +1 -1
- package/dist/lancedb-backend.d.ts +6 -0
- package/dist/lancedb-backend.d.ts.map +1 -1
- package/dist/lancedb-backend.js +53 -12
- package/dist/lancedb-backend.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -10
- package/dist/hnsw-lite.d.ts +0 -29
- package/dist/hnsw-lite.d.ts.map +0 -1
- package/dist/hnsw-lite.js +0 -240
- package/dist/hnsw-lite.js.map +0 -1
- package/dist/hybrid-backend.d.ts +0 -297
- package/dist/hybrid-backend.d.ts.map +0 -1
- package/dist/hybrid-backend.js +0 -805
- package/dist/hybrid-backend.js.map +0 -1
- package/dist/persistent-sona.d.ts +0 -144
- package/dist/persistent-sona.d.ts.map +0 -1
- package/dist/persistent-sona.js +0 -311
- package/dist/persistent-sona.js.map +0 -1
- package/dist/procedural/procedure-extractor-worker.d.ts +0 -27
- package/dist/procedural/procedure-extractor-worker.d.ts.map +0 -1
- package/dist/procedural/procedure-extractor-worker.js +0 -39
- package/dist/procedural/procedure-extractor-worker.js.map +0 -1
- package/dist/rvf-backend.d.ts +0 -53
- package/dist/rvf-backend.d.ts.map +0 -1
- package/dist/rvf-backend.js +0 -507
- package/dist/rvf-backend.js.map +0 -1
- package/dist/rvf-learning-store.d.ts +0 -141
- package/dist/rvf-learning-store.d.ts.map +0 -1
- package/dist/rvf-learning-store.js +0 -344
- package/dist/rvf-learning-store.js.map +0 -1
- package/dist/rvf-migration.d.ts +0 -45
- package/dist/rvf-migration.d.ts.map +0 -1
- package/dist/rvf-migration.js +0 -254
- package/dist/rvf-migration.js.map +0 -1
package/dist/hybrid-backend.js
DELETED
|
@@ -1,805 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HybridBackend - Combines SQLite (structured queries) + LanceDB (vector search)
|
|
3
|
-
*
|
|
4
|
-
* Per ADR-009: "HybridBackend (SQLite + LanceDB) as default"
|
|
5
|
-
* - SQLite for: Structured queries, ACID transactions, exact matches
|
|
6
|
-
* - LanceDB for: Semantic search, vector similarity, RAG
|
|
7
|
-
*
|
|
8
|
-
* @module v1/memory/hybrid-backend
|
|
9
|
-
*/
|
|
10
|
-
import { EventEmitter } from 'node:events';
|
|
11
|
-
import { SQLiteBackend } from './sqlite-backend.js';
|
|
12
|
-
import { LanceDBBackend } from './lancedb-backend.js';
|
|
13
|
-
import { MemoryGraph } from './memory-graph.js';
|
|
14
|
-
/**
|
|
15
|
-
* Default configuration
|
|
16
|
-
*/
|
|
17
|
-
const DEFAULT_CONFIG = {
|
|
18
|
-
sqlite: {},
|
|
19
|
-
lancedb: {},
|
|
20
|
-
semanticBackend: 'lancedb',
|
|
21
|
-
defaultNamespace: 'default',
|
|
22
|
-
embeddingGenerator: undefined,
|
|
23
|
-
routingStrategy: 'auto',
|
|
24
|
-
dualWrite: true,
|
|
25
|
-
semanticThreshold: 0.7,
|
|
26
|
-
hybridMaxResults: 100,
|
|
27
|
-
filterInjection: false,
|
|
28
|
-
memoragRewriter: undefined,
|
|
29
|
-
};
|
|
30
|
-
/**
|
|
31
|
-
* Structural prompt-injection patterns for filtering externally-sourced content.
|
|
32
|
-
* Source: arXiv:2302.12173, arXiv:2310.12815 — indirect prompt injection in RAG pipelines.
|
|
33
|
-
*/
|
|
34
|
-
const INJECTION_PATTERNS = [
|
|
35
|
-
/ignore\s+(all\s+)?previous\s+instructions?/i,
|
|
36
|
-
/you\s+are\s+now\s+(a|an)\s+/i,
|
|
37
|
-
/system\s*prompt\s*[:=]/i,
|
|
38
|
-
/\[\s*system\s*\]/i,
|
|
39
|
-
/\bforget\s+(your\s+)?(previous\s+)?instructions?\b/i,
|
|
40
|
-
/act\s+as\s+(if\s+you\s+(are|were)\s+)?/i,
|
|
41
|
-
/jailbreak/i,
|
|
42
|
-
/\bDAN\b/,
|
|
43
|
-
/<\s*\/?system\s*>/i,
|
|
44
|
-
];
|
|
45
|
-
/**
|
|
46
|
-
* HybridBackend Implementation
|
|
47
|
-
*
|
|
48
|
-
* Intelligently routes queries between SQLite and LanceDB:
|
|
49
|
-
* - Exact matches, prefix queries → SQLite
|
|
50
|
-
* - Semantic search, similarity → LanceDB
|
|
51
|
-
* - Complex hybrid queries → Both backends with intelligent merging
|
|
52
|
-
*/
|
|
53
|
-
export class HybridBackend extends EventEmitter {
|
|
54
|
-
sqlite; // SQLiteBackend normally; same as semantic when lancedb solo
|
|
55
|
-
semantic; // LanceDBBackend
|
|
56
|
-
config;
|
|
57
|
-
initialized = false;
|
|
58
|
-
/** Lazy MemoryGraph for HippoRAG PPR re-ranking */
|
|
59
|
-
memoryGraph = null;
|
|
60
|
-
// Performance tracking
|
|
61
|
-
stats = {
|
|
62
|
-
sqliteQueries: 0,
|
|
63
|
-
semanticQueries: 0,
|
|
64
|
-
hybridQueries: 0,
|
|
65
|
-
totalQueryTime: 0,
|
|
66
|
-
};
|
|
67
|
-
constructor(config = {}) {
|
|
68
|
-
super();
|
|
69
|
-
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
70
|
-
// Initialize semantic backend (LanceDB)
|
|
71
|
-
if (this.config.semanticBackend === 'lancedb') {
|
|
72
|
-
// Solo LanceDB mode: one backend handles all queries (structured + vector).
|
|
73
|
-
// Both this.sqlite and this.semantic point to the same instance so all
|
|
74
|
-
// routing paths (exact/prefix/tag/semantic) transparently use LanceDB.
|
|
75
|
-
// dualWrite forced off to prevent double-writes to the same instance.
|
|
76
|
-
const ldb = new LanceDBBackend({
|
|
77
|
-
...this.config.lancedb,
|
|
78
|
-
namespace: this.config.defaultNamespace,
|
|
79
|
-
embeddingGenerator: this.config.embeddingGenerator,
|
|
80
|
-
});
|
|
81
|
-
this.semantic = ldb;
|
|
82
|
-
this.sqlite = ldb;
|
|
83
|
-
this.config.dualWrite = false;
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
this.sqlite = new SQLiteBackend({
|
|
87
|
-
...this.config.sqlite,
|
|
88
|
-
defaultNamespace: this.config.defaultNamespace,
|
|
89
|
-
embeddingGenerator: this.config.embeddingGenerator,
|
|
90
|
-
});
|
|
91
|
-
this.semantic = new LanceDBBackend({
|
|
92
|
-
...this.config.lancedb,
|
|
93
|
-
namespace: this.config.defaultNamespace,
|
|
94
|
-
embeddingGenerator: this.config.embeddingGenerator,
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
// Forward events (skip duplicate wiring in solo mode)
|
|
98
|
-
const em = (b) => b;
|
|
99
|
-
em(this.sqlite).on?.('entry:stored', (d) => this.emit('sqlite:stored', d));
|
|
100
|
-
em(this.sqlite).on?.('entry:updated', (d) => this.emit('sqlite:updated', d));
|
|
101
|
-
em(this.sqlite).on?.('entry:deleted', (d) => this.emit('sqlite:deleted', d));
|
|
102
|
-
if (this.semantic !== this.sqlite) {
|
|
103
|
-
em(this.semantic).on?.('entry:stored', (d) => this.emit('semantic:stored', d));
|
|
104
|
-
em(this.semantic).on?.('entry:updated', (d) => this.emit('semantic:updated', d));
|
|
105
|
-
em(this.semantic).on?.('entry:deleted', (d) => this.emit('semantic:deleted', d));
|
|
106
|
-
em(this.semantic).on?.('cache:hit', (d) => this.emit('cache:hit', d));
|
|
107
|
-
em(this.semantic).on?.('cache:miss', (d) => this.emit('cache:miss', d));
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Initialize both backends
|
|
112
|
-
*/
|
|
113
|
-
async initialize() {
|
|
114
|
-
if (this.initialized)
|
|
115
|
-
return;
|
|
116
|
-
const inits = this.semantic === this.sqlite
|
|
117
|
-
? [this.semantic.initialize()]
|
|
118
|
-
: [this.sqlite.initialize(), this.semantic.initialize()];
|
|
119
|
-
await Promise.all(inits);
|
|
120
|
-
this.initialized = true;
|
|
121
|
-
this.emit('initialized');
|
|
122
|
-
}
|
|
123
|
-
/**
|
|
124
|
-
* Shutdown both backends
|
|
125
|
-
*/
|
|
126
|
-
async shutdown() {
|
|
127
|
-
if (!this.initialized)
|
|
128
|
-
return;
|
|
129
|
-
const shutdowns = this.semantic === this.sqlite
|
|
130
|
-
? [this.semantic.shutdown()]
|
|
131
|
-
: [this.sqlite.shutdown(), this.semantic.shutdown()];
|
|
132
|
-
await Promise.all(shutdowns);
|
|
133
|
-
this.initialized = false;
|
|
134
|
-
this.emit('shutdown');
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* Store in both backends (dual-write for consistency).
|
|
138
|
-
*
|
|
139
|
-
* A-MEM auto-linking: after writing, HNSW-searches for the top-3 most similar
|
|
140
|
-
* existing entries and adds bidirectional `references` edges between them and
|
|
141
|
-
* the new entry. Only runs when an `embeddingGenerator` is configured.
|
|
142
|
-
* Source: arXiv:2409.11987 (A-MEM — Zettelkasten-style agent memory)
|
|
143
|
-
*/
|
|
144
|
-
async store(entry) {
|
|
145
|
-
if (this.config.dualWrite) {
|
|
146
|
-
// Write to both backends in parallel
|
|
147
|
-
await Promise.all([this.sqlite.store(entry), this.semantic.store(entry)]);
|
|
148
|
-
}
|
|
149
|
-
else {
|
|
150
|
-
// Write to primary backend only
|
|
151
|
-
await this.semantic.store(entry);
|
|
152
|
-
}
|
|
153
|
-
this.emit('entry:stored', { id: entry.id });
|
|
154
|
-
// A-MEM: auto-link to top-3 semantic neighbors (non-blocking, best-effort)
|
|
155
|
-
if (typeof this.config.embeddingGenerator === 'function' && entry.content) {
|
|
156
|
-
this.aMemAutoLink(entry).catch(() => {
|
|
157
|
-
// Linking is best-effort; storage already succeeded
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* A-MEM bidirectional reference linking.
|
|
163
|
-
* Finds the top-3 semantically similar existing entries and creates
|
|
164
|
-
* `references` edges in both directions.
|
|
165
|
-
*/
|
|
166
|
-
async aMemAutoLink(newEntry) {
|
|
167
|
-
try {
|
|
168
|
-
const embedding = await this.config.embeddingGenerator(newEntry.content);
|
|
169
|
-
const neighbors = await this.semantic.search(embedding, {
|
|
170
|
-
k: 4, // fetch 4 to exclude self
|
|
171
|
-
threshold: 0.75,
|
|
172
|
-
});
|
|
173
|
-
const topNeighbors = neighbors
|
|
174
|
-
.filter(r => r.entry.id !== newEntry.id)
|
|
175
|
-
.slice(0, 3);
|
|
176
|
-
if (topNeighbors.length === 0)
|
|
177
|
-
return;
|
|
178
|
-
const neighborIds = topNeighbors.map(r => r.entry.id);
|
|
179
|
-
// Update new entry to reference neighbors
|
|
180
|
-
const newRefs = [...new Set([...(newEntry.references ?? []), ...neighborIds])];
|
|
181
|
-
await this.update(newEntry.id, { references: newRefs });
|
|
182
|
-
// Update each neighbor to back-reference the new entry
|
|
183
|
-
await Promise.all(topNeighbors.map(async ({ entry: neighbor }) => {
|
|
184
|
-
const backRefs = [...new Set([...(neighbor.references ?? []), newEntry.id])];
|
|
185
|
-
await this.update(neighbor.id, { references: backRefs });
|
|
186
|
-
}));
|
|
187
|
-
this.emit('amem:linked', { id: newEntry.id, linkedTo: neighborIds });
|
|
188
|
-
}
|
|
189
|
-
catch {
|
|
190
|
-
// Linking is best-effort
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
/**
|
|
194
|
-
* Get from semantic backend.
|
|
195
|
-
*
|
|
196
|
-
* Collaborative memory promotion: when `agentId` is supplied, also notifies
|
|
197
|
-
* the SQLite backend so it can track the read and promote the entry's
|
|
198
|
-
* AccessLevel to 'team' once 3+ distinct agents have accessed it.
|
|
199
|
-
* Source: newinnovation.md §2.7 — memory access-level promotion.
|
|
200
|
-
*/
|
|
201
|
-
async get(id, agentId) {
|
|
202
|
-
const entry = await this.semantic.get(id);
|
|
203
|
-
// Track the read in SQLite for collaborative promotion (best-effort, non-blocking).
|
|
204
|
-
// Skipped in solo-lancedb mode since this.sqlite === this.semantic (already fetched above).
|
|
205
|
-
if (agentId && entry && this.semantic !== this.sqlite) {
|
|
206
|
-
this.sqlite.get(id, agentId).catch(() => { });
|
|
207
|
-
}
|
|
208
|
-
return entry;
|
|
209
|
-
}
|
|
210
|
-
/**
|
|
211
|
-
* Get by key (SQLite optimized for exact matches)
|
|
212
|
-
*/
|
|
213
|
-
async getByKey(namespace, key) {
|
|
214
|
-
return this.sqlite.getByKey(namespace, key);
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* Update in both backends
|
|
218
|
-
*/
|
|
219
|
-
async update(id, update) {
|
|
220
|
-
if (this.config.dualWrite) {
|
|
221
|
-
// Update both backends
|
|
222
|
-
const [sqliteResult, semanticResult] = await Promise.all([
|
|
223
|
-
this.sqlite.update(id, update),
|
|
224
|
-
this.semantic.update(id, update),
|
|
225
|
-
]);
|
|
226
|
-
return semanticResult || sqliteResult;
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
return this.semantic.update(id, update);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
/**
|
|
233
|
-
* Delete from both backends
|
|
234
|
-
*/
|
|
235
|
-
async delete(id) {
|
|
236
|
-
if (this.config.dualWrite) {
|
|
237
|
-
const [sqliteResult, semanticResult] = await Promise.all([
|
|
238
|
-
this.sqlite.delete(id),
|
|
239
|
-
this.semantic.delete(id),
|
|
240
|
-
]);
|
|
241
|
-
return sqliteResult || semanticResult;
|
|
242
|
-
}
|
|
243
|
-
else {
|
|
244
|
-
return this.semantic.delete(id);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Query routing — semantic goes to LanceDB, structured to SQLite
|
|
249
|
-
*/
|
|
250
|
-
async query(query) {
|
|
251
|
-
const startTime = performance.now();
|
|
252
|
-
let results;
|
|
253
|
-
// Route based on query type
|
|
254
|
-
switch (query.type) {
|
|
255
|
-
case 'exact':
|
|
256
|
-
// SQLite optimized for exact matches
|
|
257
|
-
this.stats.sqliteQueries++;
|
|
258
|
-
results = await this.sqlite.query(query);
|
|
259
|
-
break;
|
|
260
|
-
case 'prefix':
|
|
261
|
-
// SQLite optimized for prefix queries
|
|
262
|
-
this.stats.sqliteQueries++;
|
|
263
|
-
results = await this.sqlite.query(query);
|
|
264
|
-
break;
|
|
265
|
-
case 'tag':
|
|
266
|
-
// Both can handle tags, use SQLite for structured filtering
|
|
267
|
-
this.stats.sqliteQueries++;
|
|
268
|
-
results = await this.sqlite.query(query);
|
|
269
|
-
break;
|
|
270
|
-
case 'semantic':
|
|
271
|
-
// LanceDB handles semantic search
|
|
272
|
-
this.stats.semanticQueries++;
|
|
273
|
-
results = await this.semantic.query(query);
|
|
274
|
-
break;
|
|
275
|
-
case 'hybrid':
|
|
276
|
-
// Use hybrid query combining both backends
|
|
277
|
-
this.stats.hybridQueries++;
|
|
278
|
-
results = await this.queryHybridInternal(query);
|
|
279
|
-
break;
|
|
280
|
-
default:
|
|
281
|
-
// Auto-routing based on query properties
|
|
282
|
-
results = await this.autoRoute(query);
|
|
283
|
-
}
|
|
284
|
-
const duration = performance.now() - startTime;
|
|
285
|
-
this.stats.totalQueryTime += duration;
|
|
286
|
-
this.emit('query:completed', { type: query.type, duration, count: results.length });
|
|
287
|
-
return results;
|
|
288
|
-
}
|
|
289
|
-
/**
|
|
290
|
-
* Structured queries (SQL)
|
|
291
|
-
* Routes to SQLite for optimal performance
|
|
292
|
-
*/
|
|
293
|
-
async queryStructured(query) {
|
|
294
|
-
this.stats.sqliteQueries++;
|
|
295
|
-
const memoryQuery = {
|
|
296
|
-
type: query.key ? 'exact' : query.keyPrefix ? 'prefix' : 'hybrid',
|
|
297
|
-
key: query.key,
|
|
298
|
-
keyPrefix: query.keyPrefix,
|
|
299
|
-
namespace: query.namespace,
|
|
300
|
-
ownerId: query.ownerId,
|
|
301
|
-
memoryType: query.type,
|
|
302
|
-
createdAfter: query.createdAfter,
|
|
303
|
-
createdBefore: query.createdBefore,
|
|
304
|
-
updatedAfter: query.updatedAfter,
|
|
305
|
-
updatedBefore: query.updatedBefore,
|
|
306
|
-
limit: query.limit || 100,
|
|
307
|
-
offset: query.offset || 0,
|
|
308
|
-
};
|
|
309
|
-
return this.sqlite.query(memoryQuery);
|
|
310
|
-
}
|
|
311
|
-
/**
|
|
312
|
-
* Apply injection filtering and structured field filters to a set of entries.
|
|
313
|
-
* Used by the MemoRAG path (RRF-fused results) and as a shared post-processing step.
|
|
314
|
-
*/
|
|
315
|
-
postProcessSemanticResults(entries, query) {
|
|
316
|
-
let result = entries;
|
|
317
|
-
if (this.config.filterInjection) {
|
|
318
|
-
result = result.filter(entry => {
|
|
319
|
-
const content = entry.content ?? '';
|
|
320
|
-
if (INJECTION_PATTERNS.some(rx => rx.test(content))) {
|
|
321
|
-
this.emit('injection:blocked', { id: entry.id, namespace: entry.namespace });
|
|
322
|
-
return false;
|
|
323
|
-
}
|
|
324
|
-
return true;
|
|
325
|
-
});
|
|
326
|
-
}
|
|
327
|
-
if (query.filters) {
|
|
328
|
-
const f = query.filters;
|
|
329
|
-
if (f.tags && Array.isArray(f.tags)) {
|
|
330
|
-
const requiredTags = f.tags;
|
|
331
|
-
result = result.filter(e => requiredTags.every(t => e.tags.includes(t)));
|
|
332
|
-
}
|
|
333
|
-
if (f.namespace && typeof f.namespace === 'string') {
|
|
334
|
-
result = result.filter(e => e.namespace === f.namespace);
|
|
335
|
-
}
|
|
336
|
-
if (f.type && typeof f.type === 'string' && f.type !== 'semantic') {
|
|
337
|
-
result = result.filter(e => e.type === f.type);
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
return result.slice(0, query.k || 10);
|
|
341
|
-
}
|
|
342
|
-
/**
|
|
343
|
-
* Reciprocal Rank Fusion — merge multiple ranked result lists.
|
|
344
|
-
* Score for entry i in list j: 1 / (k + rank_j(i)) where k=60 (standard).
|
|
345
|
-
* Source: Cormack et al., "Reciprocal Rank Fusion outperforms Condorcet and individual
|
|
346
|
-
* Rank Learning Methods" (SIGIR 2009). Used by MemoRAG sub-query fusion.
|
|
347
|
-
*/
|
|
348
|
-
rrfFuse(rankedLists, k = 60) {
|
|
349
|
-
const scores = new Map();
|
|
350
|
-
const byId = new Map();
|
|
351
|
-
for (const list of rankedLists) {
|
|
352
|
-
for (let rank = 0; rank < list.length; rank++) {
|
|
353
|
-
const entry = list[rank];
|
|
354
|
-
byId.set(entry.id, entry);
|
|
355
|
-
scores.set(entry.id, (scores.get(entry.id) ?? 0) + 1 / (k + rank + 1));
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
return [...byId.keys()]
|
|
359
|
-
.sort((a, b) => (scores.get(b) ?? 0) - (scores.get(a) ?? 0))
|
|
360
|
-
.map(id => byId.get(id));
|
|
361
|
-
}
|
|
362
|
-
/**
|
|
363
|
-
* Semantic queries (vector)
|
|
364
|
-
* Routes to semantic backend for ANN vector search.
|
|
365
|
-
*
|
|
366
|
-
* MemoRAG query rewriting (arXiv:2409.05591): when `memoragRewriter` is configured
|
|
367
|
-
* and `query.content` is present, generates 2-3 reformulated sub-queries, embeds
|
|
368
|
-
* each, searches HNSW independently, then fuses with Reciprocal Rank Fusion (RRF)
|
|
369
|
-
* before HippoRAG PPR re-ranking.
|
|
370
|
-
*/
|
|
371
|
-
async querySemantic(query) {
|
|
372
|
-
this.stats.semanticQueries++;
|
|
373
|
-
let searchResults = [];
|
|
374
|
-
// MemoRAG: expand the query into multiple sub-queries for richer recall
|
|
375
|
-
let memoragUsed = false;
|
|
376
|
-
if (query.content && typeof this.config.memoragRewriter === 'function') {
|
|
377
|
-
try {
|
|
378
|
-
const subQueries = await this.config.memoragRewriter(query.content);
|
|
379
|
-
if (subQueries.length > 0) {
|
|
380
|
-
// Embed all sub-queries and search in parallel
|
|
381
|
-
const subResults = await Promise.all(subQueries.map(async (sq) => {
|
|
382
|
-
if (!this.config.embeddingGenerator)
|
|
383
|
-
return [];
|
|
384
|
-
const sqEmbedding = await this.config.embeddingGenerator(sq);
|
|
385
|
-
const results = await this.semantic.search(sqEmbedding, {
|
|
386
|
-
k: (query.k || 10) * 2,
|
|
387
|
-
threshold: query.threshold || this.config.semanticThreshold,
|
|
388
|
-
filters: query.filters,
|
|
389
|
-
});
|
|
390
|
-
return results.map(r => r.entry);
|
|
391
|
-
}));
|
|
392
|
-
this.emit('memorag:rewritten', { original: query.content, subQueries, count: subResults.flat().length });
|
|
393
|
-
// RRF-fuse the sub-query results; assign to searchResults to fall through to PPR/GraphRAG
|
|
394
|
-
const fused = this.rrfFuse(subResults);
|
|
395
|
-
searchResults = fused.map(entry => ({ entry, score: 1.0, distance: 0 }));
|
|
396
|
-
memoragUsed = true;
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
catch {
|
|
400
|
-
// Rewriter failed — fall through to standard path
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
if (!memoragUsed) {
|
|
404
|
-
let embedding = query.embedding;
|
|
405
|
-
// Generate embedding if content provided
|
|
406
|
-
if (!embedding && query.content && this.config.embeddingGenerator) {
|
|
407
|
-
embedding = await this.config.embeddingGenerator(query.content);
|
|
408
|
-
}
|
|
409
|
-
if (!embedding) {
|
|
410
|
-
throw new Error('SemanticQuery requires either content or embedding');
|
|
411
|
-
}
|
|
412
|
-
searchResults = await this.semantic.search(embedding, {
|
|
413
|
-
k: (query.k || 10) * 2, // Over-fetch to account for post-filtering
|
|
414
|
-
threshold: query.threshold || this.config.semanticThreshold,
|
|
415
|
-
filters: query.filters,
|
|
416
|
-
});
|
|
417
|
-
}
|
|
418
|
-
// HippoRAG PPR re-ranking: expand one hop through MemoryEntry.references
|
|
419
|
-
// Source: https://arxiv.org/abs/2405.14831
|
|
420
|
-
let entries;
|
|
421
|
-
if (searchResults.length > 0) {
|
|
422
|
-
if (!this.memoryGraph) {
|
|
423
|
-
this.memoryGraph = new MemoryGraph();
|
|
424
|
-
// Seed graph with reference edges from search results
|
|
425
|
-
for (const r of searchResults) {
|
|
426
|
-
this.memoryGraph.addNode(r.entry);
|
|
427
|
-
for (const refId of (r.entry.references ?? [])) {
|
|
428
|
-
this.memoryGraph.addEdge(r.entry.id, refId, 'reference', 1.0);
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
else {
|
|
433
|
-
for (const r of searchResults) {
|
|
434
|
-
this.memoryGraph.addNode(r.entry);
|
|
435
|
-
for (const refId of (r.entry.references ?? [])) {
|
|
436
|
-
this.memoryGraph.addEdge(r.entry.id, refId, 'reference', 1.0);
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
// GraphRAG: compute community summaries before PPR so communities are populated.
|
|
441
|
-
// Community summaries are prepended to each entry's metadata so downstream callers
|
|
442
|
-
// can use global thematic context alongside local vector similarity.
|
|
443
|
-
// Source: https://arxiv.org/abs/2404.16130 (Microsoft GraphRAG)
|
|
444
|
-
const communitySummaries = this.memoryGraph.getCommunitySummaries(5);
|
|
445
|
-
// Build a nodeId → community summary lookup for O(1) annotation
|
|
446
|
-
const nodeToSummary = new Map();
|
|
447
|
-
for (const summary of communitySummaries) {
|
|
448
|
-
for (const nodeId of summary.topNodeIds) {
|
|
449
|
-
nodeToSummary.set(nodeId, {
|
|
450
|
-
communityId: summary.communityId,
|
|
451
|
-
nodeCount: summary.nodeCount,
|
|
452
|
-
avgPageRank: summary.avgPageRank,
|
|
453
|
-
});
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
const reranked = await this.memoryGraph.pprRerank(searchResults, this.semantic);
|
|
457
|
-
// Annotate each entry with its community ID and summary from GraphRAG
|
|
458
|
-
entries = reranked.map((r) => {
|
|
459
|
-
const commSummary = nodeToSummary.get(r.entry.id);
|
|
460
|
-
const community = r.community ?? commSummary?.communityId;
|
|
461
|
-
if (!community && !commSummary)
|
|
462
|
-
return r.entry;
|
|
463
|
-
return {
|
|
464
|
-
...r.entry,
|
|
465
|
-
metadata: {
|
|
466
|
-
...(r.entry.metadata ?? {}),
|
|
467
|
-
community,
|
|
468
|
-
...(commSummary ? {
|
|
469
|
-
communityNodeCount: commSummary.nodeCount,
|
|
470
|
-
communityAvgPageRank: commSummary.avgPageRank,
|
|
471
|
-
} : {}),
|
|
472
|
-
},
|
|
473
|
-
};
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
else {
|
|
477
|
-
entries = searchResults.map((r) => r.entry);
|
|
478
|
-
}
|
|
479
|
-
// Injection filter: remove entries whose content matches known injection signatures
|
|
480
|
-
if (this.config.filterInjection) {
|
|
481
|
-
const safe = [];
|
|
482
|
-
for (const entry of entries) {
|
|
483
|
-
const content = entry.content ?? '';
|
|
484
|
-
if (INJECTION_PATTERNS.some(rx => rx.test(content))) {
|
|
485
|
-
this.emit('injection:blocked', { id: entry.id, namespace: entry.namespace });
|
|
486
|
-
}
|
|
487
|
-
else {
|
|
488
|
-
safe.push(entry);
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
entries = safe;
|
|
492
|
-
}
|
|
493
|
-
// Apply tag/namespace/type filters
|
|
494
|
-
if (query.filters) {
|
|
495
|
-
const f = query.filters;
|
|
496
|
-
if (f.tags && Array.isArray(f.tags)) {
|
|
497
|
-
const requiredTags = f.tags;
|
|
498
|
-
entries = entries.filter((e) => requiredTags.every((t) => e.tags.includes(t)));
|
|
499
|
-
}
|
|
500
|
-
if (f.namespace && typeof f.namespace === 'string') {
|
|
501
|
-
entries = entries.filter((e) => e.namespace === f.namespace);
|
|
502
|
-
}
|
|
503
|
-
if (f.type && typeof f.type === 'string' && f.type !== 'semantic') {
|
|
504
|
-
entries = entries.filter((e) => e.type === f.type);
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
return entries.slice(0, query.k || 10);
|
|
508
|
-
}
|
|
509
|
-
/**
|
|
510
|
-
* Hybrid queries (combine both)
|
|
511
|
-
* Intelligently merges results from both backends
|
|
512
|
-
*/
|
|
513
|
-
async queryHybrid(query) {
|
|
514
|
-
this.stats.hybridQueries++;
|
|
515
|
-
const strategy = query.combineStrategy || 'semantic-first';
|
|
516
|
-
const weights = query.weights || { semantic: 0.7, structured: 0.3 };
|
|
517
|
-
// Execute both queries in parallel
|
|
518
|
-
const [semanticResults, structuredResults] = await Promise.all([
|
|
519
|
-
this.querySemantic(query.semantic),
|
|
520
|
-
query.structured ? this.queryStructured(query.structured) : Promise.resolve([]),
|
|
521
|
-
]);
|
|
522
|
-
// Combine results based on strategy
|
|
523
|
-
switch (strategy) {
|
|
524
|
-
case 'union':
|
|
525
|
-
return this.combineUnion(semanticResults, structuredResults);
|
|
526
|
-
case 'intersection':
|
|
527
|
-
return this.combineIntersection(semanticResults, structuredResults);
|
|
528
|
-
case 'semantic-first':
|
|
529
|
-
return this.combineSemanticFirst(semanticResults, structuredResults);
|
|
530
|
-
case 'structured-first':
|
|
531
|
-
return this.combineStructuredFirst(semanticResults, structuredResults);
|
|
532
|
-
default:
|
|
533
|
-
return this.combineUnion(semanticResults, structuredResults);
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
/**
|
|
537
|
-
* Semantic vector search (routes to LanceDB)
|
|
538
|
-
*/
|
|
539
|
-
async search(embedding, options) {
|
|
540
|
-
this.stats.semanticQueries++;
|
|
541
|
-
return this.semantic.search(embedding, options);
|
|
542
|
-
}
|
|
543
|
-
/**
|
|
544
|
-
* Bulk insert to both backends
|
|
545
|
-
*/
|
|
546
|
-
async bulkInsert(entries) {
|
|
547
|
-
if (this.config.dualWrite) {
|
|
548
|
-
await Promise.all([this.sqlite.bulkInsert(entries), this.semantic.bulkInsert(entries)]);
|
|
549
|
-
}
|
|
550
|
-
else {
|
|
551
|
-
await this.semantic.bulkInsert(entries);
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
/**
|
|
555
|
-
* Bulk delete from both backends
|
|
556
|
-
*/
|
|
557
|
-
async bulkDelete(ids) {
|
|
558
|
-
if (this.config.dualWrite) {
|
|
559
|
-
const [sqliteCount, semanticCount] = await Promise.all([
|
|
560
|
-
this.sqlite.bulkDelete(ids),
|
|
561
|
-
this.semantic.bulkDelete(ids),
|
|
562
|
-
]);
|
|
563
|
-
return Math.max(sqliteCount, semanticCount);
|
|
564
|
-
}
|
|
565
|
-
else {
|
|
566
|
-
return this.semantic.bulkDelete(ids);
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
/**
|
|
570
|
-
* Count entries (use SQLite for efficiency)
|
|
571
|
-
*/
|
|
572
|
-
async count(namespace) {
|
|
573
|
-
return this.sqlite.count(namespace);
|
|
574
|
-
}
|
|
575
|
-
/**
|
|
576
|
-
* List namespaces (use SQLite)
|
|
577
|
-
*/
|
|
578
|
-
async listNamespaces() {
|
|
579
|
-
return this.sqlite.listNamespaces();
|
|
580
|
-
}
|
|
581
|
-
/**
|
|
582
|
-
* Clear namespace in both backends
|
|
583
|
-
*/
|
|
584
|
-
async clearNamespace(namespace) {
|
|
585
|
-
if (this.config.dualWrite) {
|
|
586
|
-
const [sqliteCount, semanticCount] = await Promise.all([
|
|
587
|
-
this.sqlite.clearNamespace(namespace),
|
|
588
|
-
this.semantic.clearNamespace(namespace),
|
|
589
|
-
]);
|
|
590
|
-
return Math.max(sqliteCount, semanticCount);
|
|
591
|
-
}
|
|
592
|
-
else {
|
|
593
|
-
return this.semantic.clearNamespace(namespace);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
/**
|
|
597
|
-
* Get combined statistics from both backends
|
|
598
|
-
*/
|
|
599
|
-
async getStats() {
|
|
600
|
-
const solo = this.semantic === this.sqlite;
|
|
601
|
-
const [sqliteStats, semanticStats] = await Promise.all([
|
|
602
|
-
solo ? Promise.resolve(null) : this.sqlite.getStats(),
|
|
603
|
-
this.semantic.getStats(),
|
|
604
|
-
]);
|
|
605
|
-
return {
|
|
606
|
-
totalEntries: solo ? semanticStats.totalEntries : Math.max(sqliteStats.totalEntries, semanticStats.totalEntries),
|
|
607
|
-
entriesByNamespace: semanticStats.entriesByNamespace,
|
|
608
|
-
entriesByType: semanticStats.entriesByType,
|
|
609
|
-
memoryUsage: solo ? semanticStats.memoryUsage : sqliteStats.memoryUsage + semanticStats.memoryUsage,
|
|
610
|
-
hnswStats: semanticStats.hnswStats ?? {
|
|
611
|
-
vectorCount: 0, // 0 = no active HNSW index; do not fabricate from totalEntries
|
|
612
|
-
memoryUsage: 0,
|
|
613
|
-
avgSearchTime: 0,
|
|
614
|
-
buildTime: 0,
|
|
615
|
-
compressionRatio: 1.0,
|
|
616
|
-
},
|
|
617
|
-
cacheStats: semanticStats.cacheStats ?? {
|
|
618
|
-
hitRate: 0,
|
|
619
|
-
size: 0,
|
|
620
|
-
maxSize: 1000,
|
|
621
|
-
},
|
|
622
|
-
avgQueryTime: this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.semanticQueries > 0
|
|
623
|
-
? this.stats.totalQueryTime /
|
|
624
|
-
(this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.semanticQueries)
|
|
625
|
-
: 0,
|
|
626
|
-
avgSearchTime: semanticStats.avgSearchTime,
|
|
627
|
-
};
|
|
628
|
-
}
|
|
629
|
-
/**
|
|
630
|
-
* Health check for both backends
|
|
631
|
-
*/
|
|
632
|
-
async healthCheck() {
|
|
633
|
-
const solo = this.semantic === this.sqlite;
|
|
634
|
-
const [sqliteHealth, semanticHealth] = await Promise.all([
|
|
635
|
-
solo ? Promise.resolve({ status: 'healthy', issues: [], recommendations: [], components: {}, timestamp: Date.now() }) : this.sqlite.healthCheck(),
|
|
636
|
-
this.semantic.healthCheck(),
|
|
637
|
-
]);
|
|
638
|
-
const allIssues = [...sqliteHealth.issues, ...semanticHealth.issues];
|
|
639
|
-
const allRecommendations = [
|
|
640
|
-
...sqliteHealth.recommendations,
|
|
641
|
-
...semanticHealth.recommendations,
|
|
642
|
-
];
|
|
643
|
-
// Determine overall status
|
|
644
|
-
let status = 'healthy';
|
|
645
|
-
if (sqliteHealth.status === 'unhealthy' ||
|
|
646
|
-
semanticHealth.status === 'unhealthy') {
|
|
647
|
-
status = 'unhealthy';
|
|
648
|
-
}
|
|
649
|
-
else if (sqliteHealth.status === 'degraded' ||
|
|
650
|
-
semanticHealth.status === 'degraded') {
|
|
651
|
-
status = 'degraded';
|
|
652
|
-
}
|
|
653
|
-
return {
|
|
654
|
-
status,
|
|
655
|
-
components: {
|
|
656
|
-
storage: sqliteHealth.components.storage,
|
|
657
|
-
index: semanticHealth.components.index,
|
|
658
|
-
cache: semanticHealth.components.cache,
|
|
659
|
-
},
|
|
660
|
-
timestamp: Date.now(),
|
|
661
|
-
issues: allIssues,
|
|
662
|
-
recommendations: allRecommendations,
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
// ===== Private Methods =====
|
|
666
|
-
/**
|
|
667
|
-
* Auto-route queries based on properties
|
|
668
|
-
*/
|
|
669
|
-
async autoRoute(query) {
|
|
670
|
-
// If has embedding or content, use semantic search (LanceDB)
|
|
671
|
-
const hasEmbeddingGenerator = typeof this.config.embeddingGenerator === 'function';
|
|
672
|
-
if (query.embedding || (query.content && hasEmbeddingGenerator)) {
|
|
673
|
-
this.stats.semanticQueries++;
|
|
674
|
-
return this.semantic.query(query);
|
|
675
|
-
}
|
|
676
|
-
// If has exact key or prefix, use structured search (SQLite)
|
|
677
|
-
if (query.key || query.keyPrefix) {
|
|
678
|
-
this.stats.sqliteQueries++;
|
|
679
|
-
return this.sqlite.query(query);
|
|
680
|
-
}
|
|
681
|
-
// For other filters, use routing strategy
|
|
682
|
-
switch (this.config.routingStrategy) {
|
|
683
|
-
case 'sqlite-first':
|
|
684
|
-
this.stats.sqliteQueries++;
|
|
685
|
-
return this.sqlite.query(query);
|
|
686
|
-
case 'semantic-first':
|
|
687
|
-
this.stats.semanticQueries++;
|
|
688
|
-
return this.semantic.query(query);
|
|
689
|
-
case 'auto':
|
|
690
|
-
default:
|
|
691
|
-
// Default to semantic backend (LanceDB)
|
|
692
|
-
this.stats.semanticQueries++;
|
|
693
|
-
return this.semantic.query(query);
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
/**
|
|
697
|
-
* Internal hybrid query implementation
|
|
698
|
-
*/
|
|
699
|
-
async queryHybridInternal(query) {
|
|
700
|
-
// If semantic component exists, use hybrid
|
|
701
|
-
if (query.embedding || query.content) {
|
|
702
|
-
const semanticQuery = {
|
|
703
|
-
content: query.content,
|
|
704
|
-
embedding: query.embedding,
|
|
705
|
-
k: query.limit || 10,
|
|
706
|
-
threshold: query.threshold,
|
|
707
|
-
filters: query,
|
|
708
|
-
};
|
|
709
|
-
const structuredQuery = {
|
|
710
|
-
namespace: query.namespace,
|
|
711
|
-
key: query.key,
|
|
712
|
-
keyPrefix: query.keyPrefix,
|
|
713
|
-
ownerId: query.ownerId,
|
|
714
|
-
type: query.memoryType,
|
|
715
|
-
createdAfter: query.createdAfter,
|
|
716
|
-
createdBefore: query.createdBefore,
|
|
717
|
-
updatedAfter: query.updatedAfter,
|
|
718
|
-
updatedBefore: query.updatedBefore,
|
|
719
|
-
limit: query.limit,
|
|
720
|
-
offset: query.offset,
|
|
721
|
-
};
|
|
722
|
-
return this.queryHybrid({
|
|
723
|
-
semantic: semanticQuery,
|
|
724
|
-
structured: structuredQuery,
|
|
725
|
-
combineStrategy: 'semantic-first',
|
|
726
|
-
});
|
|
727
|
-
}
|
|
728
|
-
// Otherwise, route to structured
|
|
729
|
-
return this.autoRoute(query);
|
|
730
|
-
}
|
|
731
|
-
/**
|
|
732
|
-
* Combine results using union (all unique results)
|
|
733
|
-
*/
|
|
734
|
-
combineUnion(semanticResults, structuredResults) {
|
|
735
|
-
const seen = new Set();
|
|
736
|
-
const combined = [];
|
|
737
|
-
for (const entry of semanticResults) {
|
|
738
|
-
if (!seen.has(entry.id)) {
|
|
739
|
-
seen.add(entry.id);
|
|
740
|
-
combined.push(entry);
|
|
741
|
-
}
|
|
742
|
-
}
|
|
743
|
-
for (const entry of structuredResults) {
|
|
744
|
-
if (!seen.has(entry.id)) {
|
|
745
|
-
seen.add(entry.id);
|
|
746
|
-
combined.push(entry);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
return combined;
|
|
750
|
-
}
|
|
751
|
-
/**
|
|
752
|
-
* Combine results using intersection (only common results)
|
|
753
|
-
*/
|
|
754
|
-
combineIntersection(semanticResults, structuredResults) {
|
|
755
|
-
const semanticIds = new Set(semanticResults.map((e) => e.id));
|
|
756
|
-
return structuredResults.filter((e) => semanticIds.has(e.id));
|
|
757
|
-
}
|
|
758
|
-
/**
|
|
759
|
-
* Semantic-first: Prefer semantic results, add structured if not present
|
|
760
|
-
*/
|
|
761
|
-
combineSemanticFirst(semanticResults, structuredResults) {
|
|
762
|
-
const semanticIds = new Set();
|
|
763
|
-
for (const e of semanticResults)
|
|
764
|
-
semanticIds.add(e.id);
|
|
765
|
-
const additional = structuredResults.filter((e) => !semanticIds.has(e.id));
|
|
766
|
-
return [...semanticResults, ...additional];
|
|
767
|
-
}
|
|
768
|
-
/**
|
|
769
|
-
* Structured-first: Prefer structured results, add semantic if not present
|
|
770
|
-
*/
|
|
771
|
-
combineStructuredFirst(semanticResults, structuredResults) {
|
|
772
|
-
const structuredIds = new Set();
|
|
773
|
-
for (const e of structuredResults)
|
|
774
|
-
structuredIds.add(e.id);
|
|
775
|
-
const additional = semanticResults.filter((e) => !structuredIds.has(e.id));
|
|
776
|
-
return [...structuredResults, ...additional];
|
|
777
|
-
}
|
|
778
|
-
// ===== Proxy Methods — stubs for legacy controller API =====
|
|
779
|
-
/**
|
|
780
|
-
* Record feedback for a memory entry.
|
|
781
|
-
* Delegates to semantic backend's recordFeedback when available.
|
|
782
|
-
* Gracefully degrades to no-op when unavailable.
|
|
783
|
-
*/
|
|
784
|
-
async recordFeedback(_entryId, _feedback) {
|
|
785
|
-
return false; // monolean: stub — LanceDB has no feedback API
|
|
786
|
-
}
|
|
787
|
-
async verifyWitnessChain(_entryId) {
|
|
788
|
-
return { valid: false, chainLength: 0, errors: ['not supported by LanceDB backend'] };
|
|
789
|
-
}
|
|
790
|
-
async getWitnessChain(_entryId) {
|
|
791
|
-
return [];
|
|
792
|
-
}
|
|
793
|
-
// ===== Backend Access =====
|
|
794
|
-
/**
|
|
795
|
-
* Get underlying backends for advanced operations
|
|
796
|
-
*/
|
|
797
|
-
getSQLiteBackend() {
|
|
798
|
-
return this.sqlite;
|
|
799
|
-
}
|
|
800
|
-
getSemanticBackend() {
|
|
801
|
-
return this.semantic;
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
export default HybridBackend;
|
|
805
|
-
//# sourceMappingURL=hybrid-backend.js.map
|