@monoes/memory 1.0.10 → 1.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +10 -11
  2. package/dist/controller-registry.d.ts +7 -5
  3. package/dist/controller-registry.d.ts.map +1 -1
  4. package/dist/controller-registry.js +8 -6
  5. package/dist/controller-registry.js.map +1 -1
  6. package/dist/index.d.ts +38 -10
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +39 -12
  9. package/dist/index.js.map +1 -1
  10. package/dist/migration.d.ts +3 -2
  11. package/dist/migration.d.ts.map +1 -1
  12. package/dist/migration.js +3 -2
  13. package/dist/migration.js.map +1 -1
  14. package/dist/sql-backend.d.ts +81 -0
  15. package/dist/sql-backend.d.ts.map +1 -0
  16. package/dist/sql-backend.js +553 -0
  17. package/dist/sql-backend.js.map +1 -0
  18. package/dist/sql-driver.d.ts +135 -0
  19. package/dist/sql-driver.d.ts.map +1 -0
  20. package/dist/sql-driver.js +187 -0
  21. package/dist/sql-driver.js.map +1 -0
  22. package/dist/sql-schema.d.ts +74 -0
  23. package/dist/sql-schema.d.ts.map +1 -0
  24. package/dist/sql-schema.js +178 -0
  25. package/dist/sql-schema.js.map +1 -0
  26. package/dist/sqlite-backend.d.ts +16 -106
  27. package/dist/sqlite-backend.d.ts.map +1 -1
  28. package/dist/sqlite-backend.js +53 -725
  29. package/dist/sqlite-backend.js.map +1 -1
  30. package/dist/sqljs-backend.d.ts +26 -98
  31. package/dist/sqljs-backend.d.ts.map +1 -1
  32. package/dist/sqljs-backend.js +75 -609
  33. package/dist/sqljs-backend.js.map +1 -1
  34. package/dist/types.d.ts +5 -2
  35. package/dist/types.d.ts.map +1 -1
  36. package/dist/types.js +5 -2
  37. package/dist/types.js.map +1 -1
  38. package/package.json +5 -4
  39. package/dist/lancedb-backend.d.ts +0 -79
  40. package/dist/lancedb-backend.d.ts.map +0 -1
  41. package/dist/lancedb-backend.js +0 -567
  42. package/dist/lancedb-backend.js.map +0 -1
@@ -1,18 +1,27 @@
1
1
  /**
2
- * SQLite Memory Backend
2
+ * SQLite Memory Backend (better-sqlite3)
3
3
  *
4
- * Provides structured storage for memory entries using SQLite.
5
- * Optimized for ACID transactions, exact matches, and complex queries.
6
- * Part of ADR-009: Hybrid Memory Backend (SQLite + LanceDB)
4
+ * A thin driver-selecting wrapper around SqlBackend, which holds all the
5
+ * behaviour. This file previously carried a full ~930-line IMemoryBackend
6
+ * implementation that was near-duplicated in sqljs-backend.ts; the two drifted
7
+ * into different schemas and different query semantics. See sql-backend.ts.
8
+ *
9
+ * The class name and config shape are unchanged — this is the published API.
7
10
  *
8
11
  * @module v1/memory/sqlite-backend
9
12
  */
10
- import { EventEmitter } from 'node:events';
11
- import Database from 'better-sqlite3';
12
- import { cosineSimilarity } from './math-utils.js';
13
- /**
14
- * Default configuration values
15
- */
13
+ // better-sqlite3 is an OPTIONAL dependency — sql.js is the documented fallback,
14
+ // and database-provider.ts recommends it outright on Windows, where node-gyp
15
+ // routinely fails to build native modules. This import must therefore be lazy.
16
+ //
17
+ // It used to be a static top-level import, and because index.ts re-exports
18
+ // SQLiteBackend, that made `import '@monoes/memory'` throw
19
+ // ERR_MODULE_NOT_FOUND for anyone without better-sqlite3 — the whole package,
20
+ // including the pure-WASM sql.js path, was unusable in exactly the situation
21
+ // the fallback exists for. Local test suites never caught it because
22
+ // better-sqlite3 is always installed here.
23
+ import { SqlBackend } from './sql-backend.js';
24
+ import { BetterSqliteDriver } from './sql-driver.js';
16
25
  const DEFAULT_CONFIG = {
17
26
  databasePath: ':memory:',
18
27
  walMode: true,
@@ -22,729 +31,48 @@ const DEFAULT_CONFIG = {
22
31
  verbose: false,
23
32
  };
24
33
  /**
25
- * SQLite Backend for Structured Memory Storage
34
+ * SQLite backend for structured memory storage.
26
35
  *
27
- * Provides:
28
- * - ACID transactions for data consistency
29
- * - Efficient indexing for exact matches and prefix queries
30
- * - Full-text search capabilities
31
- * - Complex SQL queries with joins and aggregations
32
- * - Persistent storage with WAL mode
36
+ * Provides ACID transactions, indexed lookups, and persistent storage with
37
+ * WAL mode. Behaviour lives in SqlBackend; this class supplies the driver.
33
38
  */
34
- export class SQLiteBackend extends EventEmitter {
35
- config;
36
- db = null;
37
- initialized = false;
38
- // Performance tracking
39
- stats = {
40
- queryCount: 0,
41
- totalQueryTime: 0,
42
- writeCount: 0,
43
- totalWriteTime: 0,
44
- };
45
- // Cached prepared statements — prepared once in initialize() to avoid N+1 re-prepare
46
- stmtGetEmbedding = null;
47
- stmtInsertEntry = null;
48
- stmtInsertEmbedding = null;
49
- stmtDeleteTags = null;
50
- stmtInsertTag = null;
51
- // Debounce counter for checkAndPromoteEntry (H2: avoid full-table purge on every get)
52
- _readCount = 0;
39
+ export class SQLiteBackend extends SqlBackend {
40
+ sqliteConfig;
53
41
  constructor(config = {}) {
54
- super();
55
- this.config = { ...DEFAULT_CONFIG, ...config };
56
- }
57
- /**
58
- * Initialize the SQLite backend
59
- */
60
- async initialize() {
61
- if (this.initialized)
62
- return;
63
- // Open database connection
64
- this.db = new Database(this.config.databasePath, {
65
- verbose: this.config.verbose ? console.log : undefined,
42
+ const merged = { ...DEFAULT_CONFIG, ...config };
43
+ super({
44
+ defaultNamespace: merged.defaultNamespace,
45
+ embeddingGenerator: merged.embeddingGenerator,
46
+ maxEntries: merged.maxEntries,
47
+ verbose: merged.verbose,
66
48
  });
67
- // Enable WAL mode for better concurrency
68
- if (this.config.walMode) {
69
- this.db.pragma('journal_mode = WAL');
70
- }
71
- // Enable FK enforcement — required for ON DELETE CASCADE to fire
72
- this.db.pragma('foreign_keys = ON');
73
- // Performance optimizations
74
- if (this.config.optimize) {
75
- this.db.pragma('synchronous = NORMAL');
76
- this.db.pragma('cache_size = 10000');
77
- this.db.pragma('temp_store = MEMORY');
78
- }
79
- // Create schema
80
- this.createSchema();
81
- // Pre-compile hot-path statements to avoid N+1 re-prepare
82
- this.stmtGetEmbedding = this.db.prepare('SELECT embedding FROM memory_embeddings WHERE entry_id = ?');
83
- this.stmtInsertEntry = this.db.prepare(`
84
- INSERT OR REPLACE INTO memory_entries (
85
- id, key, content, type, namespace, tags, metadata,
86
- owner_id, access_level, created_at, updated_at, expires_at,
87
- event_at, version, "references", access_count, last_accessed_at
88
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
89
- `);
90
- this.stmtInsertEmbedding = this.db.prepare('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)');
91
- this.stmtDeleteTags = this.db.prepare('DELETE FROM memory_entry_tags WHERE entry_id = ?');
92
- this.stmtInsertTag = this.db.prepare('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)');
93
- this.initialized = true;
94
- this.emit('initialized');
95
- }
96
- /**
97
- * Shutdown the backend
98
- */
99
- async shutdown() {
100
- if (!this.initialized || !this.db)
101
- return;
102
- // Optimize database before closing
103
- if (this.config.optimize) {
104
- this.db.pragma('optimize');
105
- }
106
- this.db.close();
107
- this.db = null;
108
- this.stmtGetEmbedding = null;
109
- this.stmtInsertEntry = null;
110
- this.stmtInsertEmbedding = null;
111
- this.stmtDeleteTags = null;
112
- this.stmtInsertTag = null;
113
- this.initialized = false;
114
- this.emit('shutdown');
115
- }
116
- /**
117
- * Store a memory entry
118
- */
119
- async store(entry) {
120
- this.ensureInitialized();
121
- const startTime = performance.now();
122
- this.validateTags(entry.tags);
123
- const doStore = this.db.transaction((e) => {
124
- // Read existing embedding BEFORE INSERT OR REPLACE fires CASCADE and deletes it
125
- let embeddingToStore = e.embedding;
126
- if (!embeddingToStore) {
127
- const existingEmb = this.stmtGetEmbedding.get(e.id);
128
- if (existingEmb?.embedding) {
129
- const buf = existingEmb.embedding;
130
- embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
131
- }
132
- }
133
- this.stmtInsertEntry.run(e.id, e.key, e.content, e.type, e.namespace, JSON.stringify(e.tags), JSON.stringify(e.metadata), e.ownerId || null, e.accessLevel, e.createdAt, e.updatedAt, e.expiresAt || null, e.eventAt ?? null, e.version, JSON.stringify(e.references), e.accessCount, e.lastAccessedAt);
134
- this.stmtDeleteTags.run(e.id);
135
- for (const tag of e.tags) {
136
- this.stmtInsertTag.run(e.id, tag);
137
- }
138
- if (embeddingToStore) {
139
- this.stmtInsertEmbedding.run(e.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
140
- }
141
- });
142
- doStore(entry);
143
- const duration = performance.now() - startTime;
144
- this.stats.writeCount++;
145
- this.stats.totalWriteTime += duration;
146
- this.emit('entry:stored', { id: entry.id, duration });
147
- }
148
- /**
149
- * Get a memory entry by ID
150
- */
151
- async get(id, agentId) {
152
- this.ensureInitialized();
153
- const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
154
- const row = stmt.get(id);
155
- if (!row)
156
- return null;
157
- // Collaborative memory promotion: track per-agent reads for auto-promotion
158
- // Source: https://arxiv.org/abs/2505.18279
159
- const AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,128}$/;
160
- if (agentId && AGENT_ID_RE.test(agentId)) {
161
- try {
162
- this.db.prepare('INSERT OR IGNORE INTO agent_reads (entry_id, agent_id, read_at) VALUES (?, ?, ?)').run(id, agentId, Date.now());
163
- // Debounce: run full-table purge only every 1000 reads (H2)
164
- this._readCount++;
165
- if (this._readCount % 1000 === 0) {
166
- void this.checkAndPromoteEntry(id);
167
- }
168
- }
169
- catch { /* non-critical */ }
170
- }
171
- return this.rowToEntry(row);
172
- }
173
- /**
174
- * Auto-promote an entry's access_level to 'team' when 3+ distinct agents
175
- * have read it within the past 24 hours.
176
- * Source: https://arxiv.org/abs/2505.18279
177
- */
178
- checkAndPromoteEntry(entryId) {
179
- if (!this.db)
180
- return;
181
- const cutoff = Date.now() - 24 * 60 * 60 * 1000;
182
- // Purge stale read records to keep table size bounded
183
- this.db.prepare('DELETE FROM agent_reads WHERE read_at <= ?').run(cutoff);
184
- const countRow = this.db.prepare('SELECT COUNT(DISTINCT agent_id) as cnt FROM agent_reads WHERE entry_id = ? AND read_at > ?').get(entryId, cutoff);
185
- if ((countRow?.cnt ?? 0) >= 3) {
186
- this.db.prepare("UPDATE memory_entries SET access_level = 'team' WHERE id = ? AND access_level = 'private'").run(entryId);
187
- }
188
- }
189
- /**
190
- * Get a memory entry by key within a namespace
191
- */
192
- async getByKey(namespace, key) {
193
- this.ensureInitialized();
194
- const stmt = this.db.prepare(`
195
- SELECT * FROM memory_entries
196
- WHERE namespace = ? AND key = ?
197
- `);
198
- const row = stmt.get(namespace, key);
199
- if (!row)
200
- return null;
201
- return this.rowToEntry(row);
202
- }
203
- /**
204
- * Update a memory entry
205
- */
206
- async update(id, update) {
207
- this.ensureInitialized();
208
- const entry = await this.get(id);
209
- if (!entry)
210
- return null;
211
- // Apply updates
212
- if (update.content !== undefined)
213
- entry.content = update.content;
214
- if (update.tags !== undefined)
215
- entry.tags = update.tags;
216
- if (update.metadata !== undefined) {
217
- entry.metadata = { ...entry.metadata, ...update.metadata };
218
- }
219
- if (update.accessLevel !== undefined)
220
- entry.accessLevel = update.accessLevel;
221
- if (update.expiresAt !== undefined)
222
- entry.expiresAt = update.expiresAt;
223
- if (update.references !== undefined)
224
- entry.references = update.references;
225
- entry.updatedAt = Date.now();
226
- entry.version++;
227
- // Store updated entry
228
- await this.store(entry);
229
- this.emit('entry:updated', { id });
230
- return entry;
49
+ this.sqliteConfig = merged;
231
50
  }
232
- /**
233
- * Delete a memory entry
234
- */
235
- async delete(id) {
236
- this.ensureInitialized();
237
- const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
238
- // Explicit tag cleanup before entry delete (belt-and-suspenders with CASCADE)
239
- this.stmtDeleteTags.run(id);
240
- const result = deleteEntry.run(id);
241
- if (result.changes > 0) {
242
- this.emit('entry:deleted', { id });
243
- return true;
244
- }
245
- return false;
246
- }
247
- /**
248
- * Query memory entries with filters
249
- */
250
- async query(query) {
251
- this.ensureInitialized();
252
- const startTime = performance.now();
253
- let sql = 'SELECT * FROM memory_entries WHERE 1=1';
254
- const params = [];
255
- // Build WHERE clauses
256
- if (query.namespace) {
257
- sql += ' AND namespace = ?';
258
- params.push(query.namespace);
259
- }
260
- if (query.key) {
261
- sql += ' AND key = ?';
262
- params.push(query.key);
263
- }
264
- if (query.keyPrefix) {
265
- sql += ' AND key LIKE ?';
266
- params.push(`${query.keyPrefix}%`);
267
- }
268
- if (query.memoryType) {
269
- sql += ' AND type = ?';
270
- params.push(query.memoryType);
271
- }
272
- if (query.accessLevel) {
273
- sql += ' AND access_level = ?';
274
- params.push(query.accessLevel);
275
- }
276
- if (query.ownerId) {
277
- sql += ' AND owner_id = ?';
278
- params.push(query.ownerId);
279
- }
280
- if (query.createdAfter) {
281
- sql += ' AND created_at >= ?';
282
- params.push(query.createdAfter);
283
- }
284
- if (query.createdBefore) {
285
- sql += ' AND created_at <= ?';
286
- params.push(query.createdBefore);
287
- }
288
- if (query.updatedAfter) {
289
- sql += ' AND updated_at >= ?';
290
- params.push(query.updatedAfter);
291
- }
292
- if (query.updatedBefore) {
293
- sql += ' AND updated_at <= ?';
294
- params.push(query.updatedBefore);
295
- }
296
- // Bi-temporal event-time filters (arXiv:2501.13956 — Zep/Graphiti)
297
- if (query.eventAfter) {
298
- sql += ' AND event_at >= ?';
299
- params.push(query.eventAfter);
300
- }
301
- if (query.eventBefore) {
302
- sql += ' AND event_at <= ?';
303
- params.push(query.eventBefore);
304
- }
305
- if (!query.includeExpired) {
306
- sql += ' AND (expires_at IS NULL OR expires_at > ?)';
307
- params.push(Date.now());
308
- }
309
- // Tag filtering — uses indexed memory_entry_tags table (no json_each scan)
310
- if (query.tags && query.tags.length > 0) {
311
- this.validateTags(query.tags);
312
- const tagPlaceholders = query.tags.map(() => '?').join(', ');
313
- sql += ` AND EXISTS (
314
- SELECT 1 FROM memory_entry_tags t
315
- WHERE t.entry_id = memory_entries.id AND t.tag IN (${tagPlaceholders})
316
- )`;
317
- params.push(...query.tags);
318
- }
319
- // Pagination — always enforce a cap to prevent full-table scans
320
- const MAX_QUERY_LIMIT = 10_000;
321
- const effectiveLimit = Math.min(Math.max(1, query.limit ?? MAX_QUERY_LIMIT), MAX_QUERY_LIMIT);
322
- const colMap = {
323
- createdAt: 'created_at', updatedAt: 'updated_at',
324
- lastAccessedAt: 'last_accessed_at', accessCount: 'access_count', key: 'key',
325
- };
326
- const orderCol = (query.sortField && query.sortField !== 'score' && colMap[query.sortField])
327
- ? colMap[query.sortField]
328
- : 'created_at';
329
- const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
330
- sql += ` ORDER BY ${orderCol} ${orderDir}`;
331
- sql += ' LIMIT ?';
332
- params.push(effectiveLimit);
333
- if (query.offset) {
334
- sql += ' OFFSET ?';
335
- params.push(query.offset);
336
- }
337
- const stmt = this.db.prepare(sql);
338
- const rows = stmt.all(...params);
339
- const results = rows.map((row) => this.rowToEntry(row));
340
- const duration = performance.now() - startTime;
341
- this.stats.queryCount++;
342
- this.stats.totalQueryTime += duration;
343
- return results;
344
- }
345
- /**
346
- * Semantic vector search (not optimized for SQLite, returns empty)
347
- * Use HybridBackend for semantic search with LanceDB
348
- */
349
- async search(embedding, options) {
350
- this.ensureInitialized();
351
- // Brute-force cosine over stored embeddings, namespace-scoped and
352
- // TTL-filtered in SQL. iterate() streams rows one at a time instead of
353
- // materializing the whole embedding table (.all() held every BLOB in
354
- // memory at once). At second-brain scale (10^3–10^5 vectors of 384 dims)
355
- // this is a few tens of ms; swap in the HNSW index if a profiled
356
- // workload outgrows it.
357
- const ns = options.filters?.namespace;
358
- const rows = this.db.prepare(`
359
- SELECT e.*, emb.embedding AS _emb
360
- FROM memory_entries e
361
- JOIN memory_embeddings emb ON emb.entry_id = e.id
362
- WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
363
- ${ns ? 'AND e.namespace = ?' : ''}
364
- `).iterate(...(ns ? [Date.now(), ns] : [Date.now()]));
365
- const results = [];
366
- for (const row of rows) {
367
- const buf = row._emb;
368
- if (!buf || buf.byteLength % 4 !== 0)
369
- continue;
370
- const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
371
- if (vec.length !== embedding.length)
372
- continue;
373
- const similarity = cosineSimilarity(embedding, vec);
374
- if (options.threshold !== undefined && similarity < options.threshold)
375
- continue;
376
- delete row._emb;
377
- results.push({ entry: this.rowToEntry(row), score: similarity, distance: 1 - similarity });
378
- }
379
- results.sort((a, b) => b.score - a.score);
380
- return results.slice(0, options.k);
381
- }
382
- /**
383
- * Bulk insert entries
384
- */
385
- async bulkInsert(entries) {
386
- this.ensureInitialized();
387
- const transaction = this.db.transaction((entries) => {
388
- for (const entry of entries) {
389
- this.storeSync(entry);
390
- }
391
- });
392
- transaction(entries);
393
- this.emit('bulk:inserted', { count: entries.length });
394
- }
395
- /**
396
- * Bulk delete entries
397
- */
398
- async bulkDelete(ids) {
399
- this.ensureInitialized();
400
- const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
401
- const transaction = this.db.transaction((ids) => {
402
- let deleted = 0;
403
- for (const id of ids) {
404
- this.stmtDeleteTags.run(id);
405
- const result = deleteEntry.run(id);
406
- if (result.changes > 0)
407
- deleted++;
408
- }
409
- return deleted;
410
- });
411
- return transaction(ids);
412
- }
413
- /**
414
- * Get entry count
415
- */
416
- async count(namespace) {
417
- this.ensureInitialized();
418
- let sql = 'SELECT COUNT(*) as count FROM memory_entries';
419
- const params = [];
420
- if (namespace) {
421
- sql += ' WHERE namespace = ?';
422
- params.push(namespace);
423
- }
424
- const stmt = this.db.prepare(sql);
425
- const result = stmt.get(...params);
426
- return result.count;
427
- }
428
- /**
429
- * List all namespaces
430
- */
431
- async listNamespaces() {
432
- this.ensureInitialized();
433
- const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
434
- const rows = stmt.all();
435
- return rows.map((row) => row.namespace);
436
- }
437
- /**
438
- * Clear all entries in a namespace
439
- */
440
- async clearNamespace(namespace) {
441
- this.ensureInitialized();
442
- const deleteTags = this.db.prepare(`
443
- DELETE FROM memory_entry_tags
444
- WHERE entry_id IN (SELECT id FROM memory_entries WHERE namespace = ?)
445
- `);
446
- const deleteEntries = this.db.prepare('DELETE FROM memory_entries WHERE namespace = ?');
447
- const deleteOrphanEmbeddings = this.db.prepare(`
448
- DELETE FROM memory_embeddings
449
- WHERE entry_id NOT IN (SELECT id FROM memory_entries)
450
- `);
451
- let changes = 0;
452
- this.db.transaction(() => {
453
- deleteTags.run(namespace);
454
- changes = deleteEntries.run(namespace).changes;
455
- deleteOrphanEmbeddings.run();
456
- })();
457
- return changes;
458
- }
459
- /**
460
- * Get backend statistics
461
- */
462
- async getStats() {
463
- this.ensureInitialized();
464
- // Count by namespace
465
- const namespaceStmt = this.db.prepare(`
466
- SELECT namespace, COUNT(*) as count
467
- FROM memory_entries
468
- GROUP BY namespace
469
- `);
470
- const namespaceRows = namespaceStmt.all();
471
- const entriesByNamespace = {};
472
- for (const row of namespaceRows) {
473
- entriesByNamespace[row.namespace] = row.count;
474
- }
475
- // Count by type
476
- const typeStmt = this.db.prepare(`
477
- SELECT type, COUNT(*) as count
478
- FROM memory_entries
479
- GROUP BY type
480
- `);
481
- const typeRows = typeStmt.all();
482
- const entriesByType = {
483
- episodic: 0,
484
- semantic: 0,
485
- working: 0,
486
- cache: 0,
487
- };
488
- for (const row of typeRows) {
489
- entriesByType[row.type] = row.count;
490
- }
491
- // Get database size
492
- const pageCount = this.db.pragma('page_count', { simple: true });
493
- const pageSize = this.db.pragma('page_size', { simple: true });
494
- const memoryUsage = pageCount * pageSize;
495
- const totalEntries = await this.count();
496
- return {
497
- totalEntries,
498
- entriesByNamespace,
499
- entriesByType,
500
- memoryUsage,
501
- avgQueryTime: this.stats.queryCount > 0
502
- ? this.stats.totalQueryTime / this.stats.queryCount
503
- : 0,
504
- avgSearchTime: 0, // Not applicable for SQLite
505
- };
506
- }
507
- /**
508
- * Perform health check
509
- */
510
- async healthCheck() {
511
- const issues = [];
512
- const recommendations = [];
513
- if (!this.initialized || !this.db) {
514
- return {
515
- status: 'unhealthy',
516
- components: {
517
- storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' },
518
- index: { status: 'healthy', latency: 0 },
519
- cache: { status: 'healthy', latency: 0 },
520
- },
521
- timestamp: Date.now(),
522
- issues: ['Backend not initialized'],
523
- recommendations: ['Call initialize() before using'],
524
- };
525
- }
526
- // Check database integrity
527
- let storageHealth;
51
+ async openDriver() {
52
+ let Database;
528
53
  try {
529
- const integrityCheck = this.db.pragma('integrity_check', { simple: true });
530
- if (integrityCheck === 'ok') {
531
- storageHealth = { status: 'healthy', latency: 0 };
532
- }
533
- else {
534
- issues.push('Database integrity check failed');
535
- recommendations.push('Run VACUUM to repair database');
536
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' };
537
- }
54
+ const mod = await import('better-sqlite3');
55
+ Database = mod.default;
538
56
  }
539
57
  catch (error) {
540
- issues.push('Failed to check database integrity');
541
- storageHealth = { status: 'unhealthy', latency: 0, message: String(error) };
542
- }
543
- // Check utilization
544
- const totalEntries = await this.count();
545
- const utilizationPercent = (totalEntries / this.config.maxEntries) * 100;
546
- if (utilizationPercent > 95) {
547
- issues.push('Storage utilization critical (>95%)');
548
- recommendations.push('Cleanup old data or increase maxEntries');
549
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' };
550
- }
551
- else if (utilizationPercent > 80) {
552
- issues.push('Storage utilization high (>80%)');
553
- recommendations.push('Consider cleanup');
554
- if (storageHealth.status === 'healthy') {
555
- storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' };
556
- }
557
- }
558
- const status = storageHealth.status === 'unhealthy'
559
- ? 'unhealthy'
560
- : storageHealth.status === 'degraded'
561
- ? 'degraded'
562
- : 'healthy';
563
- return {
564
- status,
565
- components: {
566
- storage: storageHealth,
567
- index: { status: 'healthy', latency: 0 },
568
- cache: { status: 'healthy', latency: 0 },
569
- },
570
- timestamp: Date.now(),
571
- issues,
572
- recommendations,
573
- };
574
- }
575
- // ===== Private Methods =====
576
- // Every tag write below is parameterized — validation is shape sanity, not
577
- // injection defense. `src:<absolute path>` provenance tags are first-class
578
- // in the knowledge pipeline and paths legitimately contain @, #, spaces,
579
- // unicode, parentheses… so src: tags only exclude control characters.
580
- // (The sql.js backend has never validated tags at all; a strict charset
581
- // here made store success silently depend on which backend loaded.)
582
- static TAG_RE = /^[a-zA-Z0-9_\-.:/~ ]+$/;
583
- static SRC_TAG_RE = /^src:[^\x00-\x1f\x7f]+$/;
584
- static MAX_TAG_LEN = 512;
585
- validateTags(tags) {
586
- for (const tag of tags) {
587
- const ok = typeof tag === 'string' && tag.length <= SQLiteBackend.MAX_TAG_LEN
588
- && (tag.startsWith('src:') ? SQLiteBackend.SRC_TAG_RE.test(tag) : SQLiteBackend.TAG_RE.test(tag));
589
- if (!ok)
590
- throw new Error(`Invalid tag format: "${String(tag).slice(0, 80)}"`);
591
- }
592
- }
593
- ensureInitialized() {
594
- if (!this.initialized || !this.db) {
595
- throw new Error('SQLiteBackend not initialized. Call initialize() first.');
596
- }
597
- }
598
- createSchema() {
599
- if (!this.db)
600
- return;
601
- // Main entries table
602
- this.db.exec(`
603
- CREATE TABLE IF NOT EXISTS memory_entries (
604
- id TEXT PRIMARY KEY,
605
- key TEXT NOT NULL,
606
- content TEXT NOT NULL,
607
- type TEXT NOT NULL,
608
- namespace TEXT NOT NULL,
609
- tags TEXT NOT NULL,
610
- metadata TEXT NOT NULL,
611
- owner_id TEXT,
612
- access_level TEXT NOT NULL,
613
- created_at INTEGER NOT NULL,
614
- updated_at INTEGER NOT NULL,
615
- expires_at INTEGER,
616
- event_at INTEGER,
617
- version INTEGER NOT NULL,
618
- "references" TEXT NOT NULL,
619
- access_count INTEGER NOT NULL,
620
- last_accessed_at INTEGER NOT NULL
621
- );
622
-
623
- CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace);
624
- CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key);
625
- CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type);
626
- CREATE INDEX IF NOT EXISTS idx_owner_id ON memory_entries(owner_id);
627
- CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at);
628
- CREATE INDEX IF NOT EXISTS idx_updated_at ON memory_entries(updated_at);
629
- CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at);
630
-
631
- CREATE TABLE IF NOT EXISTS memory_embeddings (
632
- entry_id TEXT PRIMARY KEY,
633
- embedding BLOB,
634
- FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
635
- );
636
- `);
637
- // UNIQUE(namespace,key): the sql.js schema has always enforced it; this
638
- // backend only had a plain index, so repeated stores under the same key
639
- // accumulated duplicates and getByKey returned an arbitrary one. Existing
640
- // databases may already hold duplicates — dedupe (keep newest) before the
641
- // unique index can be created, then upgrade the index.
642
- try {
643
- this.db.exec(`
644
- DELETE FROM memory_entries WHERE id NOT IN (
645
- SELECT id FROM (
646
- SELECT id, ROW_NUMBER() OVER (PARTITION BY namespace, key ORDER BY updated_at DESC, id DESC) AS rn
647
- FROM memory_entries
648
- ) WHERE rn = 1
649
- );
650
- DROP INDEX IF EXISTS idx_namespace_key;
651
- CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key_unique ON memory_entries(namespace, key);
652
- `);
653
- }
654
- catch {
655
- // Very old SQLite without window functions: keep the plain index rather
656
- // than failing initialization — dup accumulation is the lesser evil.
657
- this.db.exec('CREATE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key);');
658
- }
659
- this.db.exec(`
660
-
661
- CREATE TABLE IF NOT EXISTS agent_reads (
662
- entry_id TEXT NOT NULL,
663
- agent_id TEXT NOT NULL,
664
- read_at INTEGER NOT NULL,
665
- UNIQUE(entry_id, agent_id)
666
- );
667
- CREATE INDEX IF NOT EXISTS idx_agent_reads_entry ON agent_reads(entry_id);
668
- CREATE INDEX IF NOT EXISTS idx_agent_reads_at ON agent_reads(read_at);
669
-
670
- CREATE TABLE IF NOT EXISTS memory_entry_tags (
671
- entry_id TEXT NOT NULL,
672
- tag TEXT NOT NULL,
673
- PRIMARY KEY (entry_id, tag),
674
- FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
675
- );
676
- CREATE INDEX IF NOT EXISTS idx_entry_tag ON memory_entry_tags(tag, entry_id);
677
- `);
678
- // One-time backfill gated by user_version — wrapped in a transaction to hold
679
- // a write lock and prevent double-backfill under concurrent process startup.
680
- this.db.transaction(() => {
681
- const schemaVersion = this.db.pragma('user_version', { simple: true });
682
- if (schemaVersion < 1) {
683
- this.db.exec(`
684
- INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag)
685
- SELECT memory_entries.id, t.value
686
- FROM memory_entries, json_each(tags) AS t
687
- WHERE json_valid(tags) AND t.value IS NOT NULL AND t.value != '';
688
- `);
689
- this.db.pragma('user_version = 1');
690
- }
691
- })();
692
- }
693
- rowToEntry(row) {
694
- // Get embedding if exists — use pre-compiled statement to avoid N+1 re-prepare
695
- let embedding;
696
- const embeddingRow = this.stmtGetEmbedding.get(row.id);
697
- if (embeddingRow && embeddingRow.embedding) {
698
- // Slice only the intended byte range — Buffer.from(existingBuffer) for copies
699
- // under Node's 8KB pool threshold returns a view into the SHARED pool
700
- // ArrayBuffer at a nonzero offset, so `.buffer` alone spans unrelated pool
701
- // memory (silent data corruption). byteOffset/byteLength scope it correctly.
702
- const buf = embeddingRow.embedding;
703
- embedding = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
704
- }
705
- return {
706
- id: row.id,
707
- key: row.key,
708
- content: row.content,
709
- embedding,
710
- type: row.type,
711
- namespace: row.namespace,
712
- tags: JSON.parse(row.tags),
713
- metadata: JSON.parse(row.metadata),
714
- ownerId: row.owner_id,
715
- accessLevel: row.access_level,
716
- createdAt: row.created_at,
717
- updatedAt: row.updated_at,
718
- expiresAt: row.expires_at,
719
- eventAt: row.event_at ?? undefined,
720
- version: row.version,
721
- references: JSON.parse(row.references),
722
- accessCount: row.access_count,
723
- lastAccessedAt: row.last_accessed_at,
724
- };
725
- }
726
- /**
727
- * Synchronous store for use in transactions
728
- */
729
- storeSync(entry) {
730
- this.validateTags(entry.tags);
731
- // Read existing embedding before INSERT OR REPLACE cascades-deletes it
732
- let embeddingToStore = entry.embedding;
733
- if (!embeddingToStore) {
734
- const existingEmb = this.stmtGetEmbedding.get(entry.id);
735
- if (existingEmb?.embedding) {
736
- const buf = Buffer.from(existingEmb.embedding);
737
- embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
738
- }
739
- }
740
- this.stmtInsertEntry.run(entry.id, entry.key, entry.content, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.eventAt ?? null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt);
741
- this.stmtDeleteTags.run(entry.id);
742
- for (const tag of entry.tags) {
743
- this.stmtInsertTag.run(entry.id, tag);
744
- }
745
- if (embeddingToStore) {
746
- this.stmtInsertEmbedding.run(entry.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
747
- }
58
+ // Reaching here means this backend was chosen explicitly while the native
59
+ // module is unavailable. Say so, instead of surfacing a bare
60
+ // ERR_MODULE_NOT_FOUND from deep inside the driver.
61
+ throw new Error('SQLiteBackend requires the optional dependency better-sqlite3, which is not installed ' +
62
+ `(${error.message}). Use SqlJsBackend, or select the 'sql.js' provider, ` +
63
+ 'for environments where the native module cannot be built.');
64
+ }
65
+ const db = new Database(this.sqliteConfig.databasePath, {
66
+ verbose: this.sqliteConfig.verbose ? console.log : undefined,
67
+ });
68
+ if (this.sqliteConfig.walMode)
69
+ db.pragma('journal_mode = WAL');
70
+ if (this.sqliteConfig.optimize) {
71
+ db.pragma('synchronous = NORMAL');
72
+ db.pragma('cache_size = 10000');
73
+ db.pragma('temp_store = MEMORY');
74
+ }
75
+ return new BetterSqliteDriver(db);
748
76
  }
749
77
  }
750
78
  export default SQLiteBackend;