@monoes/memory 1.0.9 → 1.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +10 -11
  2. package/dist/controller-registry.d.ts +7 -5
  3. package/dist/controller-registry.d.ts.map +1 -1
  4. package/dist/controller-registry.js +8 -6
  5. package/dist/controller-registry.js.map +1 -1
  6. package/dist/index.d.ts +38 -10
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +39 -12
  9. package/dist/index.js.map +1 -1
  10. package/dist/migration.d.ts +3 -2
  11. package/dist/migration.d.ts.map +1 -1
  12. package/dist/migration.js +3 -2
  13. package/dist/migration.js.map +1 -1
  14. package/dist/sql-backend.d.ts +81 -0
  15. package/dist/sql-backend.d.ts.map +1 -0
  16. package/dist/sql-backend.js +553 -0
  17. package/dist/sql-backend.js.map +1 -0
  18. package/dist/sql-driver.d.ts +135 -0
  19. package/dist/sql-driver.d.ts.map +1 -0
  20. package/dist/sql-driver.js +187 -0
  21. package/dist/sql-driver.js.map +1 -0
  22. package/dist/sql-schema.d.ts +74 -0
  23. package/dist/sql-schema.d.ts.map +1 -0
  24. package/dist/sql-schema.js +178 -0
  25. package/dist/sql-schema.js.map +1 -0
  26. package/dist/sqlite-backend.d.ts +16 -105
  27. package/dist/sqlite-backend.d.ts.map +1 -1
  28. package/dist/sqlite-backend.js +32 -723
  29. package/dist/sqlite-backend.js.map +1 -1
  30. package/dist/sqljs-backend.d.ts +26 -98
  31. package/dist/sqljs-backend.d.ts.map +1 -1
  32. package/dist/sqljs-backend.js +75 -609
  33. package/dist/sqljs-backend.js.map +1 -1
  34. package/dist/types.d.ts +5 -2
  35. package/dist/types.d.ts.map +1 -1
  36. package/dist/types.js +5 -2
  37. package/dist/types.js.map +1 -1
  38. package/package.json +5 -4
  39. package/dist/lancedb-backend.d.ts +0 -79
  40. package/dist/lancedb-backend.d.ts.map +0 -1
  41. package/dist/lancedb-backend.js +0 -567
  42. package/dist/lancedb-backend.js.map +0 -1
@@ -1,18 +1,18 @@
1
1
  /**
2
- * SQLite Memory Backend
2
+ * SQLite Memory Backend (better-sqlite3)
3
3
  *
4
- * Provides structured storage for memory entries using SQLite.
5
- * Optimized for ACID transactions, exact matches, and complex queries.
6
- * Part of ADR-009: Hybrid Memory Backend (SQLite + LanceDB)
4
+ * A thin driver-selecting wrapper around SqlBackend, which holds all the
5
+ * behaviour. This file previously carried a full ~930-line IMemoryBackend
6
+ * implementation that was near-duplicated in sqljs-backend.ts; the two drifted
7
+ * into different schemas and different query semantics. See sql-backend.ts.
8
+ *
9
+ * The class name and config shape are unchanged — this is the published API.
7
10
  *
8
11
  * @module v1/memory/sqlite-backend
9
12
  */
10
- import { EventEmitter } from 'node:events';
11
13
  import Database from 'better-sqlite3';
12
- import { cosineSimilarity } from './math-utils.js';
13
- /**
14
- * Default configuration values
15
- */
14
+ import { SqlBackend } from './sql-backend.js';
15
+ import { BetterSqliteDriver } from './sql-driver.js';
16
16
  const DEFAULT_CONFIG = {
17
17
  databasePath: ':memory:',
18
18
  walMode: true,
@@ -22,726 +22,35 @@ const DEFAULT_CONFIG = {
22
22
  verbose: false,
23
23
  };
24
24
  /**
25
- * SQLite Backend for Structured Memory Storage
25
+ * SQLite backend for structured memory storage.
26
26
  *
27
- * Provides:
28
- * - ACID transactions for data consistency
29
- * - Efficient indexing for exact matches and prefix queries
30
- * - Full-text search capabilities
31
- * - Complex SQL queries with joins and aggregations
32
- * - Persistent storage with WAL mode
27
+ * Provides ACID transactions, indexed lookups, and persistent storage with
28
+ * WAL mode. Behaviour lives in SqlBackend; this class supplies the driver.
33
29
  */
34
- export class SQLiteBackend extends EventEmitter {
35
- config;
36
- db = null;
37
- initialized = false;
38
- // Performance tracking
39
- stats = {
40
- queryCount: 0,
41
- totalQueryTime: 0,
42
- writeCount: 0,
43
- totalWriteTime: 0,
44
- };
45
- // Cached prepared statements — prepared once in initialize() to avoid N+1 re-prepare
46
- stmtGetEmbedding = null;
47
- stmtInsertEntry = null;
48
- stmtInsertEmbedding = null;
49
- stmtDeleteTags = null;
50
- stmtInsertTag = null;
51
- // Debounce counter for checkAndPromoteEntry (H2: avoid full-table purge on every get)
52
- _readCount = 0;
30
+ export class SQLiteBackend extends SqlBackend {
31
+ sqliteConfig;
53
32
  constructor(config = {}) {
54
- super();
55
- this.config = { ...DEFAULT_CONFIG, ...config };
56
- }
57
- /**
58
- * Initialize the SQLite backend
59
- */
60
- async initialize() {
61
- if (this.initialized)
62
- return;
63
- // Open database connection
64
- this.db = new Database(this.config.databasePath, {
65
- verbose: this.config.verbose ? console.log : undefined,
33
+ const merged = { ...DEFAULT_CONFIG, ...config };
34
+ super({
35
+ defaultNamespace: merged.defaultNamespace,
36
+ embeddingGenerator: merged.embeddingGenerator,
37
+ maxEntries: merged.maxEntries,
38
+ verbose: merged.verbose,
66
39
  });
67
- // Enable WAL mode for better concurrency
68
- if (this.config.walMode) {
69
- this.db.pragma('journal_mode = WAL');
70
- }
71
- // Enable FK enforcement — required for ON DELETE CASCADE to fire
72
- this.db.pragma('foreign_keys = ON');
73
- // Performance optimizations
74
- if (this.config.optimize) {
75
- this.db.pragma('synchronous = NORMAL');
76
- this.db.pragma('cache_size = 10000');
77
- this.db.pragma('temp_store = MEMORY');
78
- }
79
- // Create schema
80
- this.createSchema();
81
- // Pre-compile hot-path statements to avoid N+1 re-prepare
82
- this.stmtGetEmbedding = this.db.prepare('SELECT embedding FROM memory_embeddings WHERE entry_id = ?');
83
- this.stmtInsertEntry = this.db.prepare(`
84
- INSERT OR REPLACE INTO memory_entries (
85
- id, key, content, type, namespace, tags, metadata,
86
- owner_id, access_level, created_at, updated_at, expires_at,
87
- event_at, version, "references", access_count, last_accessed_at
88
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
89
- `);
90
- this.stmtInsertEmbedding = this.db.prepare('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)');
91
- this.stmtDeleteTags = this.db.prepare('DELETE FROM memory_entry_tags WHERE entry_id = ?');
92
- this.stmtInsertTag = this.db.prepare('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)');
93
- this.initialized = true;
94
- this.emit('initialized');
95
- }
96
- /**
97
- * Shutdown the backend
98
- */
99
- async shutdown() {
100
- if (!this.initialized || !this.db)
101
- return;
102
- // Optimize database before closing
103
- if (this.config.optimize) {
104
- this.db.pragma('optimize');
105
- }
106
- this.db.close();
107
- this.db = null;
108
- this.stmtGetEmbedding = null;
109
- this.stmtInsertEntry = null;
110
- this.stmtInsertEmbedding = null;
111
- this.stmtDeleteTags = null;
112
- this.stmtInsertTag = null;
113
- this.initialized = false;
114
- this.emit('shutdown');
40
+ this.sqliteConfig = merged;
115
41
  }
116
- /**
117
- * Store a memory entry
118
- */
119
- async store(entry) {
120
- this.ensureInitialized();
121
- const startTime = performance.now();
122
- this.validateTags(entry.tags);
123
- const doStore = this.db.transaction((e) => {
124
- // Read existing embedding BEFORE INSERT OR REPLACE fires CASCADE and deletes it
125
- let embeddingToStore = e.embedding;
126
- if (!embeddingToStore) {
127
- const existingEmb = this.stmtGetEmbedding.get(e.id);
128
- if (existingEmb?.embedding) {
129
- const buf = existingEmb.embedding;
130
- embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
131
- }
132
- }
133
- this.stmtInsertEntry.run(e.id, e.key, e.content, e.type, e.namespace, JSON.stringify(e.tags), JSON.stringify(e.metadata), e.ownerId || null, e.accessLevel, e.createdAt, e.updatedAt, e.expiresAt || null, e.eventAt ?? null, e.version, JSON.stringify(e.references), e.accessCount, e.lastAccessedAt);
134
- this.stmtDeleteTags.run(e.id);
135
- for (const tag of e.tags) {
136
- this.stmtInsertTag.run(e.id, tag);
137
- }
138
- if (embeddingToStore) {
139
- this.stmtInsertEmbedding.run(e.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
140
- }
42
+ async openDriver() {
43
+ const db = new Database(this.sqliteConfig.databasePath, {
44
+ verbose: this.sqliteConfig.verbose ? console.log : undefined,
141
45
  });
142
- doStore(entry);
143
- const duration = performance.now() - startTime;
144
- this.stats.writeCount++;
145
- this.stats.totalWriteTime += duration;
146
- this.emit('entry:stored', { id: entry.id, duration });
147
- }
148
- /**
149
- * Get a memory entry by ID
150
- */
151
- async get(id, agentId) {
152
- this.ensureInitialized();
153
- const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
154
- const row = stmt.get(id);
155
- if (!row)
156
- return null;
157
- // Collaborative memory promotion: track per-agent reads for auto-promotion
158
- // Source: https://arxiv.org/abs/2505.18279
159
- const AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,128}$/;
160
- if (agentId && AGENT_ID_RE.test(agentId)) {
161
- try {
162
- this.db.prepare('INSERT OR IGNORE INTO agent_reads (entry_id, agent_id, read_at) VALUES (?, ?, ?)').run(id, agentId, Date.now());
163
- // Debounce: run full-table purge only every 1000 reads (H2)
164
- this._readCount++;
165
- if (this._readCount % 1000 === 0) {
166
- void this.checkAndPromoteEntry(id);
167
- }
168
- }
169
- catch { /* non-critical */ }
170
- }
171
- return this.rowToEntry(row);
172
- }
173
- /**
174
- * Auto-promote an entry's access_level to 'team' when 3+ distinct agents
175
- * have read it within the past 24 hours.
176
- * Source: https://arxiv.org/abs/2505.18279
177
- */
178
- checkAndPromoteEntry(entryId) {
179
- if (!this.db)
180
- return;
181
- const cutoff = Date.now() - 24 * 60 * 60 * 1000;
182
- // Purge stale read records to keep table size bounded
183
- this.db.prepare('DELETE FROM agent_reads WHERE read_at <= ?').run(cutoff);
184
- const countRow = this.db.prepare('SELECT COUNT(DISTINCT agent_id) as cnt FROM agent_reads WHERE entry_id = ? AND read_at > ?').get(entryId, cutoff);
185
- if ((countRow?.cnt ?? 0) >= 3) {
186
- this.db.prepare("UPDATE memory_entries SET access_level = 'team' WHERE id = ? AND access_level = 'private'").run(entryId);
187
- }
188
- }
189
- /**
190
- * Get a memory entry by key within a namespace
191
- */
192
- async getByKey(namespace, key) {
193
- this.ensureInitialized();
194
- const stmt = this.db.prepare(`
195
- SELECT * FROM memory_entries
196
- WHERE namespace = ? AND key = ?
197
- `);
198
- const row = stmt.get(namespace, key);
199
- if (!row)
200
- return null;
201
- return this.rowToEntry(row);
202
- }
203
- /**
204
- * Update a memory entry
205
- */
206
- async update(id, update) {
207
- this.ensureInitialized();
208
- const entry = await this.get(id);
209
- if (!entry)
210
- return null;
211
- // Apply updates
212
- if (update.content !== undefined)
213
- entry.content = update.content;
214
- if (update.tags !== undefined)
215
- entry.tags = update.tags;
216
- if (update.metadata !== undefined) {
217
- entry.metadata = { ...entry.metadata, ...update.metadata };
218
- }
219
- if (update.accessLevel !== undefined)
220
- entry.accessLevel = update.accessLevel;
221
- if (update.expiresAt !== undefined)
222
- entry.expiresAt = update.expiresAt;
223
- if (update.references !== undefined)
224
- entry.references = update.references;
225
- entry.updatedAt = Date.now();
226
- entry.version++;
227
- // Store updated entry
228
- await this.store(entry);
229
- this.emit('entry:updated', { id });
230
- return entry;
231
- }
232
- /**
233
- * Delete a memory entry
234
- */
235
- async delete(id) {
236
- this.ensureInitialized();
237
- const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
238
- // Explicit tag cleanup before entry delete (belt-and-suspenders with CASCADE)
239
- this.stmtDeleteTags.run(id);
240
- const result = deleteEntry.run(id);
241
- if (result.changes > 0) {
242
- this.emit('entry:deleted', { id });
243
- return true;
244
- }
245
- return false;
246
- }
247
- /**
248
- * Query memory entries with filters
249
- */
250
- async query(query) {
251
- this.ensureInitialized();
252
- const startTime = performance.now();
253
- let sql = 'SELECT * FROM memory_entries WHERE 1=1';
254
- const params = [];
255
- // Build WHERE clauses
256
- if (query.namespace) {
257
- sql += ' AND namespace = ?';
258
- params.push(query.namespace);
259
- }
260
- if (query.key) {
261
- sql += ' AND key = ?';
262
- params.push(query.key);
263
- }
264
- if (query.keyPrefix) {
265
- sql += ' AND key LIKE ?';
266
- params.push(`${query.keyPrefix}%`);
267
- }
268
- if (query.memoryType) {
269
- sql += ' AND type = ?';
270
- params.push(query.memoryType);
271
- }
272
- if (query.accessLevel) {
273
- sql += ' AND access_level = ?';
274
- params.push(query.accessLevel);
275
- }
276
- if (query.ownerId) {
277
- sql += ' AND owner_id = ?';
278
- params.push(query.ownerId);
279
- }
280
- if (query.createdAfter) {
281
- sql += ' AND created_at >= ?';
282
- params.push(query.createdAfter);
283
- }
284
- if (query.createdBefore) {
285
- sql += ' AND created_at <= ?';
286
- params.push(query.createdBefore);
287
- }
288
- if (query.updatedAfter) {
289
- sql += ' AND updated_at >= ?';
290
- params.push(query.updatedAfter);
291
- }
292
- if (query.updatedBefore) {
293
- sql += ' AND updated_at <= ?';
294
- params.push(query.updatedBefore);
295
- }
296
- // Bi-temporal event-time filters (arXiv:2501.13956 — Zep/Graphiti)
297
- if (query.eventAfter) {
298
- sql += ' AND event_at >= ?';
299
- params.push(query.eventAfter);
300
- }
301
- if (query.eventBefore) {
302
- sql += ' AND event_at <= ?';
303
- params.push(query.eventBefore);
304
- }
305
- if (!query.includeExpired) {
306
- sql += ' AND (expires_at IS NULL OR expires_at > ?)';
307
- params.push(Date.now());
308
- }
309
- // Tag filtering — uses indexed memory_entry_tags table (no json_each scan)
310
- if (query.tags && query.tags.length > 0) {
311
- this.validateTags(query.tags);
312
- const tagPlaceholders = query.tags.map(() => '?').join(', ');
313
- sql += ` AND EXISTS (
314
- SELECT 1 FROM memory_entry_tags t
315
- WHERE t.entry_id = memory_entries.id AND t.tag IN (${tagPlaceholders})
316
- )`;
317
- params.push(...query.tags);
318
- }
319
- // Pagination — always enforce a cap to prevent full-table scans
320
- const MAX_QUERY_LIMIT = 10_000;
321
- const effectiveLimit = Math.min(Math.max(1, query.limit ?? MAX_QUERY_LIMIT), MAX_QUERY_LIMIT);
322
- const colMap = {
323
- createdAt: 'created_at', updatedAt: 'updated_at',
324
- lastAccessedAt: 'last_accessed_at', accessCount: 'access_count', key: 'key',
325
- };
326
- const orderCol = (query.sortField && query.sortField !== 'score' && colMap[query.sortField])
327
- ? colMap[query.sortField]
328
- : 'created_at';
329
- const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
330
- sql += ` ORDER BY ${orderCol} ${orderDir}`;
331
- sql += ' LIMIT ?';
332
- params.push(effectiveLimit);
333
- if (query.offset) {
334
- sql += ' OFFSET ?';
335
- params.push(query.offset);
336
- }
337
- const stmt = this.db.prepare(sql);
338
- const rows = stmt.all(...params);
339
- const results = rows.map((row) => this.rowToEntry(row));
340
- const duration = performance.now() - startTime;
341
- this.stats.queryCount++;
342
- this.stats.totalQueryTime += duration;
343
- return results;
344
- }
345
- /**
346
- * Semantic vector search (not optimized for SQLite, returns empty)
347
- * Use HybridBackend for semantic search with LanceDB
348
- */
349
- async search(embedding, options) {
350
- this.ensureInitialized();
351
- // Brute-force cosine over stored embeddings, namespace-scoped and
352
- // TTL-filtered in SQL. iterate() streams rows one at a time instead of
353
- // materializing the whole embedding table (.all() held every BLOB in
354
- // memory at once). At second-brain scale (10^3–10^5 vectors of 384 dims)
355
- // this is a few tens of ms; swap in the HNSW index if a profiled
356
- // workload outgrows it.
357
- const ns = options.filters?.namespace;
358
- const rows = this.db.prepare(`
359
- SELECT e.*, emb.embedding AS _emb
360
- FROM memory_entries e
361
- JOIN memory_embeddings emb ON emb.entry_id = e.id
362
- WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
363
- ${ns ? 'AND e.namespace = ?' : ''}
364
- `).iterate(...(ns ? [Date.now(), ns] : [Date.now()]));
365
- const results = [];
366
- for (const row of rows) {
367
- const buf = row._emb;
368
- if (!buf || buf.byteLength % 4 !== 0)
369
- continue;
370
- const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
371
- if (vec.length !== embedding.length)
372
- continue;
373
- const similarity = cosineSimilarity(embedding, vec);
374
- if (options.threshold !== undefined && similarity < options.threshold)
375
- continue;
376
- delete row._emb;
377
- results.push({ entry: this.rowToEntry(row), score: similarity, distance: 1 - similarity });
378
- }
379
- results.sort((a, b) => b.score - a.score);
380
- return results.slice(0, options.k);
381
- }
382
- /**
383
- * Bulk insert entries
384
- */
385
- async bulkInsert(entries) {
386
- this.ensureInitialized();
387
- const transaction = this.db.transaction((entries) => {
388
- for (const entry of entries) {
389
- this.storeSync(entry);
390
- }
391
- });
392
- transaction(entries);
393
- this.emit('bulk:inserted', { count: entries.length });
394
- }
395
- /**
396
- * Bulk delete entries
397
- */
398
- async bulkDelete(ids) {
399
- this.ensureInitialized();
400
- const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
401
- const transaction = this.db.transaction((ids) => {
402
- let deleted = 0;
403
- for (const id of ids) {
404
- this.stmtDeleteTags.run(id);
405
- const result = deleteEntry.run(id);
406
- if (result.changes > 0)
407
- deleted++;
408
- }
409
- return deleted;
410
- });
411
- return transaction(ids);
412
- }
413
- /**
414
- * Get entry count
415
- */
416
- async count(namespace) {
417
- this.ensureInitialized();
418
- let sql = 'SELECT COUNT(*) as count FROM memory_entries';
419
- const params = [];
420
- if (namespace) {
421
- sql += ' WHERE namespace = ?';
422
- params.push(namespace);
423
- }
424
- const stmt = this.db.prepare(sql);
425
- const result = stmt.get(...params);
426
- return result.count;
427
- }
428
- /**
429
- * List all namespaces
430
- */
431
- async listNamespaces() {
432
- this.ensureInitialized();
433
- const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
434
- const rows = stmt.all();
435
- return rows.map((row) => row.namespace);
436
- }
437
- /**
438
- * Clear all entries in a namespace
439
- */
440
- async clearNamespace(namespace) {
441
- this.ensureInitialized();
442
- const deleteTags = this.db.prepare(`
443
- DELETE FROM memory_entry_tags
444
- WHERE entry_id IN (SELECT id FROM memory_entries WHERE namespace = ?)
445
- `);
446
- const deleteEntries = this.db.prepare('DELETE FROM memory_entries WHERE namespace = ?');
447
- const deleteOrphanEmbeddings = this.db.prepare(`
448
- DELETE FROM memory_embeddings
449
- WHERE entry_id NOT IN (SELECT id FROM memory_entries)
450
- `);
451
- let changes = 0;
452
- this.db.transaction(() => {
453
- deleteTags.run(namespace);
454
- changes = deleteEntries.run(namespace).changes;
455
- deleteOrphanEmbeddings.run();
456
- })();
457
- return changes;
458
- }
459
- /**
460
- * Get backend statistics
461
- */
462
- async getStats() {
463
- this.ensureInitialized();
464
- // Count by namespace
465
- const namespaceStmt = this.db.prepare(`
466
- SELECT namespace, COUNT(*) as count
467
- FROM memory_entries
468
- GROUP BY namespace
469
- `);
470
- const namespaceRows = namespaceStmt.all();
471
- const entriesByNamespace = {};
472
- for (const row of namespaceRows) {
473
- entriesByNamespace[row.namespace] = row.count;
474
- }
475
- // Count by type
476
- const typeStmt = this.db.prepare(`
477
- SELECT type, COUNT(*) as count
478
- FROM memory_entries
479
- GROUP BY type
480
- `);
481
- const typeRows = typeStmt.all();
482
- const entriesByType = {
483
- episodic: 0,
484
- semantic: 0,
485
- working: 0,
486
- cache: 0,
487
- };
488
- for (const row of typeRows) {
489
- entriesByType[row.type] = row.count;
490
- }
491
- // Get database size
492
- const pageCount = this.db.pragma('page_count', { simple: true });
493
- const pageSize = this.db.pragma('page_size', { simple: true });
494
- const memoryUsage = pageCount * pageSize;
495
- const totalEntries = await this.count();
496
- return {
497
- totalEntries,
498
- entriesByNamespace,
499
- entriesByType,
500
- memoryUsage,
501
- avgQueryTime: this.stats.queryCount > 0
502
- ? this.stats.totalQueryTime / this.stats.queryCount
503
- : 0,
504
- avgSearchTime: 0, // Not applicable for SQLite
505
- };
506
- }
507
- /**
508
- * Perform health check
509
- */
510
- async healthCheck() {
511
- const issues = [];
512
- const recommendations = [];
513
- if (!this.initialized || !this.db) {
514
- return {
515
- status: 'unhealthy',
516
- components: {
517
- storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' },
518
- index: { status: 'healthy', latency: 0 },
519
- cache: { status: 'healthy', latency: 0 },
520
- },
521
- timestamp: Date.now(),
522
- issues: ['Backend not initialized'],
523
- recommendations: ['Call initialize() before using'],
524
- };
525
- }
526
- // Check database integrity
527
- let storageHealth;
528
- try {
529
- const integrityCheck = this.db.pragma('integrity_check', { simple: true });
530
- if (integrityCheck === 'ok') {
531
- storageHealth = { status: 'healthy', latency: 0 };
532
- }
533
- else {
534
- issues.push('Database integrity check failed');
535
- recommendations.push('Run VACUUM to repair database');
536
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' };
537
- }
538
- }
539
- catch (error) {
540
- issues.push('Failed to check database integrity');
541
- storageHealth = { status: 'unhealthy', latency: 0, message: String(error) };
542
- }
543
- // Check utilization
544
- const totalEntries = await this.count();
545
- const utilizationPercent = (totalEntries / this.config.maxEntries) * 100;
546
- if (utilizationPercent > 95) {
547
- issues.push('Storage utilization critical (>95%)');
548
- recommendations.push('Cleanup old data or increase maxEntries');
549
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' };
550
- }
551
- else if (utilizationPercent > 80) {
552
- issues.push('Storage utilization high (>80%)');
553
- recommendations.push('Consider cleanup');
554
- if (storageHealth.status === 'healthy') {
555
- storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' };
556
- }
557
- }
558
- const status = storageHealth.status === 'unhealthy'
559
- ? 'unhealthy'
560
- : storageHealth.status === 'degraded'
561
- ? 'degraded'
562
- : 'healthy';
563
- return {
564
- status,
565
- components: {
566
- storage: storageHealth,
567
- index: { status: 'healthy', latency: 0 },
568
- cache: { status: 'healthy', latency: 0 },
569
- },
570
- timestamp: Date.now(),
571
- issues,
572
- recommendations,
573
- };
574
- }
575
- // ===== Private Methods =====
576
- // `/`, `~`, and spaces admitted: `src:<absolute path>` provenance tags are a
577
- // first-class part of the knowledge pipeline, and every tag write below is
578
- // parameterized — the regex is shape sanity, not injection defense. The
579
- // sql.js backend has never validated tags at all, so the strict form also
580
- // made store success silently depend on which backend loaded.
581
- static TAG_RE = /^[a-zA-Z0-9_\-.:/~ ]+$/;
582
- static MAX_TAG_LEN = 512;
583
- validateTags(tags) {
584
- for (const tag of tags) {
585
- if (typeof tag !== 'string' || tag.length > SQLiteBackend.MAX_TAG_LEN || !SQLiteBackend.TAG_RE.test(tag)) {
586
- throw new Error(`Invalid tag format: "${String(tag).slice(0, 80)}"`);
587
- }
588
- }
589
- }
590
- ensureInitialized() {
591
- if (!this.initialized || !this.db) {
592
- throw new Error('SQLiteBackend not initialized. Call initialize() first.');
593
- }
594
- }
595
- createSchema() {
596
- if (!this.db)
597
- return;
598
- // Main entries table
599
- this.db.exec(`
600
- CREATE TABLE IF NOT EXISTS memory_entries (
601
- id TEXT PRIMARY KEY,
602
- key TEXT NOT NULL,
603
- content TEXT NOT NULL,
604
- type TEXT NOT NULL,
605
- namespace TEXT NOT NULL,
606
- tags TEXT NOT NULL,
607
- metadata TEXT NOT NULL,
608
- owner_id TEXT,
609
- access_level TEXT NOT NULL,
610
- created_at INTEGER NOT NULL,
611
- updated_at INTEGER NOT NULL,
612
- expires_at INTEGER,
613
- event_at INTEGER,
614
- version INTEGER NOT NULL,
615
- "references" TEXT NOT NULL,
616
- access_count INTEGER NOT NULL,
617
- last_accessed_at INTEGER NOT NULL
618
- );
619
-
620
- CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace);
621
- CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key);
622
- CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type);
623
- CREATE INDEX IF NOT EXISTS idx_owner_id ON memory_entries(owner_id);
624
- CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at);
625
- CREATE INDEX IF NOT EXISTS idx_updated_at ON memory_entries(updated_at);
626
- CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at);
627
-
628
- CREATE TABLE IF NOT EXISTS memory_embeddings (
629
- entry_id TEXT PRIMARY KEY,
630
- embedding BLOB,
631
- FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
632
- );
633
- `);
634
- // UNIQUE(namespace,key): the sql.js schema has always enforced it; this
635
- // backend only had a plain index, so repeated stores under the same key
636
- // accumulated duplicates and getByKey returned an arbitrary one. Existing
637
- // databases may already hold duplicates — dedupe (keep newest) before the
638
- // unique index can be created, then upgrade the index.
639
- try {
640
- this.db.exec(`
641
- DELETE FROM memory_entries WHERE id NOT IN (
642
- SELECT id FROM (
643
- SELECT id, ROW_NUMBER() OVER (PARTITION BY namespace, key ORDER BY updated_at DESC, id DESC) AS rn
644
- FROM memory_entries
645
- ) WHERE rn = 1
646
- );
647
- DROP INDEX IF EXISTS idx_namespace_key;
648
- CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key_unique ON memory_entries(namespace, key);
649
- `);
650
- }
651
- catch {
652
- // Very old SQLite without window functions: keep the plain index rather
653
- // than failing initialization — dup accumulation is the lesser evil.
654
- this.db.exec('CREATE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key);');
655
- }
656
- this.db.exec(`
657
-
658
- CREATE TABLE IF NOT EXISTS agent_reads (
659
- entry_id TEXT NOT NULL,
660
- agent_id TEXT NOT NULL,
661
- read_at INTEGER NOT NULL,
662
- UNIQUE(entry_id, agent_id)
663
- );
664
- CREATE INDEX IF NOT EXISTS idx_agent_reads_entry ON agent_reads(entry_id);
665
- CREATE INDEX IF NOT EXISTS idx_agent_reads_at ON agent_reads(read_at);
666
-
667
- CREATE TABLE IF NOT EXISTS memory_entry_tags (
668
- entry_id TEXT NOT NULL,
669
- tag TEXT NOT NULL,
670
- PRIMARY KEY (entry_id, tag),
671
- FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
672
- );
673
- CREATE INDEX IF NOT EXISTS idx_entry_tag ON memory_entry_tags(tag, entry_id);
674
- `);
675
- // One-time backfill gated by user_version — wrapped in a transaction to hold
676
- // a write lock and prevent double-backfill under concurrent process startup.
677
- this.db.transaction(() => {
678
- const schemaVersion = this.db.pragma('user_version', { simple: true });
679
- if (schemaVersion < 1) {
680
- this.db.exec(`
681
- INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag)
682
- SELECT memory_entries.id, t.value
683
- FROM memory_entries, json_each(tags) AS t
684
- WHERE json_valid(tags) AND t.value IS NOT NULL AND t.value != '';
685
- `);
686
- this.db.pragma('user_version = 1');
687
- }
688
- })();
689
- }
690
- rowToEntry(row) {
691
- // Get embedding if exists — use pre-compiled statement to avoid N+1 re-prepare
692
- let embedding;
693
- const embeddingRow = this.stmtGetEmbedding.get(row.id);
694
- if (embeddingRow && embeddingRow.embedding) {
695
- // Slice only the intended byte range — Buffer.from(existingBuffer) for copies
696
- // under Node's 8KB pool threshold returns a view into the SHARED pool
697
- // ArrayBuffer at a nonzero offset, so `.buffer` alone spans unrelated pool
698
- // memory (silent data corruption). byteOffset/byteLength scope it correctly.
699
- const buf = embeddingRow.embedding;
700
- embedding = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
701
- }
702
- return {
703
- id: row.id,
704
- key: row.key,
705
- content: row.content,
706
- embedding,
707
- type: row.type,
708
- namespace: row.namespace,
709
- tags: JSON.parse(row.tags),
710
- metadata: JSON.parse(row.metadata),
711
- ownerId: row.owner_id,
712
- accessLevel: row.access_level,
713
- createdAt: row.created_at,
714
- updatedAt: row.updated_at,
715
- expiresAt: row.expires_at,
716
- eventAt: row.event_at ?? undefined,
717
- version: row.version,
718
- references: JSON.parse(row.references),
719
- accessCount: row.access_count,
720
- lastAccessedAt: row.last_accessed_at,
721
- };
722
- }
723
- /**
724
- * Synchronous store for use in transactions
725
- */
726
- storeSync(entry) {
727
- this.validateTags(entry.tags);
728
- // Read existing embedding before INSERT OR REPLACE cascades-deletes it
729
- let embeddingToStore = entry.embedding;
730
- if (!embeddingToStore) {
731
- const existingEmb = this.stmtGetEmbedding.get(entry.id);
732
- if (existingEmb?.embedding) {
733
- const buf = Buffer.from(existingEmb.embedding);
734
- embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
735
- }
736
- }
737
- this.stmtInsertEntry.run(entry.id, entry.key, entry.content, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.eventAt ?? null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt);
738
- this.stmtDeleteTags.run(entry.id);
739
- for (const tag of entry.tags) {
740
- this.stmtInsertTag.run(entry.id, tag);
741
- }
742
- if (embeddingToStore) {
743
- this.stmtInsertEmbedding.run(entry.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
744
- }
46
+ if (this.sqliteConfig.walMode)
47
+ db.pragma('journal_mode = WAL');
48
+ if (this.sqliteConfig.optimize) {
49
+ db.pragma('synchronous = NORMAL');
50
+ db.pragma('cache_size = 10000');
51
+ db.pragma('temp_store = MEMORY');
52
+ }
53
+ return new BetterSqliteDriver(db);
745
54
  }
746
55
  }
747
56
  export default SQLiteBackend;