@monoes/memory 1.0.19 → 1.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/hnsw-distance.d.ts +31 -0
  2. package/dist/hnsw-distance.d.ts.map +1 -0
  3. package/dist/hnsw-distance.js +106 -0
  4. package/dist/hnsw-distance.js.map +1 -0
  5. package/dist/hnsw-graph-ops.d.ts +29 -0
  6. package/dist/hnsw-graph-ops.d.ts.map +1 -0
  7. package/dist/hnsw-graph-ops.js +96 -0
  8. package/dist/hnsw-graph-ops.js.map +1 -0
  9. package/dist/hnsw-heap.d.ts +45 -0
  10. package/dist/hnsw-heap.d.ts.map +1 -0
  11. package/dist/hnsw-heap.js +155 -0
  12. package/dist/hnsw-heap.js.map +1 -0
  13. package/dist/hnsw-index.d.ts +18 -43
  14. package/dist/hnsw-index.d.ts.map +1 -1
  15. package/dist/hnsw-index.js +21 -463
  16. package/dist/hnsw-index.js.map +1 -1
  17. package/dist/hnsw-quantizer.d.ts +28 -0
  18. package/dist/hnsw-quantizer.d.ts.map +1 -0
  19. package/dist/hnsw-quantizer.js +100 -0
  20. package/dist/hnsw-quantizer.js.map +1 -0
  21. package/dist/hnsw-serialize.d.ts +39 -0
  22. package/dist/hnsw-serialize.d.ts.map +1 -0
  23. package/dist/hnsw-serialize.js +54 -0
  24. package/dist/hnsw-serialize.js.map +1 -0
  25. package/dist/migration-loaders.d.ts +38 -0
  26. package/dist/migration-loaders.d.ts.map +1 -0
  27. package/dist/migration-loaders.js +269 -0
  28. package/dist/migration-loaders.js.map +1 -0
  29. package/dist/migration-transform.d.ts +21 -0
  30. package/dist/migration-transform.d.ts.map +1 -0
  31. package/dist/migration-transform.js +124 -0
  32. package/dist/migration-transform.js.map +1 -0
  33. package/dist/migration.d.ts +38 -22
  34. package/dist/migration.d.ts.map +1 -1
  35. package/dist/migration.js +35 -348
  36. package/dist/migration.js.map +1 -1
  37. package/dist/sql-backend-ann.d.ts +46 -0
  38. package/dist/sql-backend-ann.d.ts.map +1 -0
  39. package/dist/sql-backend-ann.js +120 -0
  40. package/dist/sql-backend-ann.js.map +1 -0
  41. package/dist/sql-backend-introspect.d.ts +17 -0
  42. package/dist/sql-backend-introspect.d.ts.map +1 -0
  43. package/dist/sql-backend-introspect.js +114 -0
  44. package/dist/sql-backend-introspect.js.map +1 -0
  45. package/dist/sql-backend-reads.d.ts +53 -0
  46. package/dist/sql-backend-reads.d.ts.map +1 -0
  47. package/dist/sql-backend-reads.js +295 -0
  48. package/dist/sql-backend-reads.js.map +1 -0
  49. package/dist/sql-backend-writes.d.ts +64 -0
  50. package/dist/sql-backend-writes.d.ts.map +1 -0
  51. package/dist/sql-backend-writes.js +270 -0
  52. package/dist/sql-backend-writes.js.map +1 -0
  53. package/dist/sql-backend.d.ts +39 -96
  54. package/dist/sql-backend.d.ts.map +1 -1
  55. package/dist/sql-backend.js +49 -706
  56. package/dist/sql-backend.js.map +1 -1
  57. package/dist/types-backend.d.ts +187 -0
  58. package/dist/types-backend.d.ts.map +1 -0
  59. package/dist/types-backend.js +8 -0
  60. package/dist/types-backend.js.map +1 -0
  61. package/dist/types-core.d.ts +180 -0
  62. package/dist/types-core.d.ts.map +1 -0
  63. package/dist/types-core.js +8 -0
  64. package/dist/types-core.js.map +1 -0
  65. package/dist/types-misc.d.ts +172 -0
  66. package/dist/types-misc.d.ts.map +1 -0
  67. package/dist/types-misc.js +56 -0
  68. package/dist/types-misc.js.map +1 -0
  69. package/dist/types.d.ts +8 -516
  70. package/dist/types.d.ts.map +1 -1
  71. package/dist/types.js +5 -49
  72. package/dist/types.js.map +1 -1
  73. package/package.json +2 -2
@@ -13,22 +13,31 @@
13
13
  * sql-driver.ts; the canonical schema and the legacy-data migration live in
14
14
  * sql-schema.ts. This file contains only behaviour.
15
15
  *
16
+ * File-size sweep: the class's method BODIES are grouped into sibling
17
+ * modules — ANN/HNSW index management in sql-backend-ann.ts, the
18
+ * store/update/delete family in sql-backend-writes.ts, the get/query/search
19
+ * family in sql-backend-reads.ts, and stats/health/persist in
20
+ * sql-backend-introspect.ts. Each stays a real method here (same signature,
21
+ * same visibility) so callers and TypeScript see no difference; the method
22
+ * body is just `return theSiblingImpl.call(this, ...args)`. Fields those
23
+ * bodies read/write moved from `private` to `protected` (compile-time only;
24
+ * no runtime change) — TypeScript allows a standalone function typed
25
+ * `this: SqlBackend` to reach `protected` members but not `private` ones, so
26
+ * `protected` is the minimum visibility the split needs.
27
+ *
16
28
  * @module v1/memory/sql-backend
17
29
  */
18
30
  import { EventEmitter } from 'node:events';
19
- import { existsSync, readFileSync } from 'node:fs';
20
- import { join } from 'node:path';
21
- import { writeFileAtomicSync } from './atomic-file.js';
22
- import { HNSWIndex } from './hnsw-index.js';
23
- import { cosineSimilarity } from './math-utils.js';
31
+ import { sqlBackendAnnMethods } from './sql-backend-ann.js';
32
+ import { sqlBackendIntrospectMethods } from './sql-backend-introspect.js';
33
+ import { sqlBackendReadMethods } from './sql-backend-reads.js';
34
+ import { sqlBackendWriteMethods } from './sql-backend-writes.js';
24
35
  import { hasFTS5Table, initializeSchema } from './sql-schema.js';
25
36
  const DEFAULT_CONFIG = {
26
37
  defaultNamespace: 'default',
27
38
  maxEntries: 1_000_000,
28
39
  verbose: false,
29
40
  };
30
- /** Cap on votes/rows pulled in one go, guarding against unbounded memory use. */
31
- const MAX_QUERY_LIMIT = 10_000;
32
41
  export class SqlBackend extends EventEmitter {
33
42
  config;
34
43
  driver = null;
@@ -65,103 +74,16 @@ export class SqlBackend extends EventEmitter {
65
74
  getAnnCacheDir() {
66
75
  return null;
67
76
  }
77
+ // Bodies live in sql-backend-ann.ts (file-size sweep) — thin wrappers here
78
+ // keep the same call signatures and visibility other methods rely on.
68
79
  annCachePath() {
69
- const dir = this.getAnnCacheDir();
70
- return dir ? join(dir, 'hnsw-index.json') : null;
80
+ return sqlBackendAnnMethods.annCachePath.call(this);
71
81
  }
72
- /**
73
- * Staleness fingerprint for the ANN cache: row count alone misses an
74
- * in-place embedding update (same id, re-embedded content — the count
75
- * doesn't change), which would otherwise leave the cached graph serving a
76
- * stale vector for that entry indefinitely. `updated_at` is bumped on
77
- * every store() call (including updates to an existing id), so pairing
78
- * count with MAX(updated_at) catches that case too.
79
- */
80
82
  countEmbeddedActiveEntries() {
81
- const row = this.driver?.get(`SELECT COUNT(*) as c, COALESCE(MAX(e.updated_at), 0) as m FROM memory_entries e
82
- JOIN memory_embeddings emb ON emb.entry_id = e.id
83
- WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)`, [Date.now()]);
84
- return { count: row?.c ?? 0, maxUpdatedAt: row?.m ?? 0 };
83
+ return sqlBackendAnnMethods.countEmbeddedActiveEntries.call(this);
85
84
  }
86
- /**
87
- * Returns a ready-to-search ANN index for the given embedding dimensions,
88
- * or null when the corpus is below ANN_THRESHOLD (brute force stays the
89
- * search path). Tries, in order: the process-lifetime cache, a valid
90
- * on-disk cache (skips the DB read + graph build entirely), then a full
91
- * rebuild from memory_embeddings (writing a fresh on-disk cache for next
92
- * time).
93
- */
94
85
  async getAnnIndex(dimensions, force = false) {
95
- const { count, maxUpdatedAt } = this.countEmbeddedActiveEntries();
96
- if (!force && count < SqlBackend.ANN_THRESHOLD)
97
- return null;
98
- if (!force &&
99
- this._annIndex &&
100
- this._annDimensions === dimensions &&
101
- this._annBuiltForCount === count &&
102
- this._annBuiltForMaxUpdatedAt === maxUpdatedAt) {
103
- return { index: this._annIndex, entries: this._annEntries };
104
- }
105
- const cachePath = this.annCachePath();
106
- if (!force && cachePath && existsSync(cachePath)) {
107
- try {
108
- const parsed = JSON.parse(readFileSync(cachePath, 'utf8'));
109
- if (parsed.entryCount === count &&
110
- parsed.maxUpdatedAt === maxUpdatedAt &&
111
- parsed.dimensions === dimensions) {
112
- const index = HNSWIndex.deserialize(parsed.index);
113
- const entries = new Map(parsed.entries);
114
- this._annIndex = index;
115
- this._annDimensions = dimensions;
116
- this._annBuiltForCount = count;
117
- this._annBuiltForMaxUpdatedAt = maxUpdatedAt;
118
- this._annEntries = entries;
119
- return { index, entries };
120
- }
121
- }
122
- catch {
123
- // Corrupt or incompatible cache — fall through to a full rebuild.
124
- }
125
- }
126
- const rows = this.driver?.iterate(`SELECT e.*, emb.embedding AS _emb
127
- FROM memory_entries e
128
- JOIN memory_embeddings emb ON emb.entry_id = e.id
129
- WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)`, [Date.now()]) ?? [];
130
- const index = new HNSWIndex({ dimensions, metric: 'cosine' });
131
- const entries = new Map();
132
- const points = [];
133
- for (const row of rows) {
134
- const buf = row._emb;
135
- if (!buf || buf.byteLength % 4 !== 0)
136
- continue;
137
- const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
138
- if (vec.length !== dimensions)
139
- continue;
140
- const entry = this.rowToEntry(row);
141
- points.push({ id: entry.id, vector: vec });
142
- entries.set(entry.id, entry);
143
- }
144
- await index.rebuild(points);
145
- this._annIndex = index;
146
- this._annDimensions = dimensions;
147
- this._annBuiltForCount = count;
148
- this._annBuiltForMaxUpdatedAt = maxUpdatedAt;
149
- this._annEntries = entries;
150
- if (cachePath) {
151
- try {
152
- writeFileAtomicSync(cachePath, JSON.stringify({
153
- entryCount: count,
154
- maxUpdatedAt,
155
- dimensions,
156
- index: index.serialize(),
157
- entries: Array.from(entries.entries()),
158
- }));
159
- }
160
- catch {
161
- // Best-effort — a failed cache write just means the next cold start rebuilds.
162
- }
163
- }
164
- return { index, entries };
86
+ return sqlBackendAnnMethods.getAnnIndex.call(this, dimensions, force);
165
87
  }
166
88
  /**
167
89
  * Diagnostics for `memory search --build-hnsw` / status reporting. Read-only
@@ -235,655 +157,76 @@ export class SqlBackend extends EventEmitter {
235
157
  this.initialized = false;
236
158
  this.emit('shutdown');
237
159
  }
238
- // ===== Writes ============================================================
160
+ // ===== Writes =============================================================
161
+ // Bodies live in sql-backend-writes.ts (file-size sweep).
239
162
  async store(entry) {
240
- this.ensureInitialized();
241
- const startTime = performance.now();
242
- this.validateTags(entry.tags);
243
- this.driver?.transaction(() => this.storeSync(entry));
244
- const duration = performance.now() - startTime;
245
- this.stats.writeCount++;
246
- this.stats.totalWriteTime += duration;
247
- this.emit('entry:stored', { id: entry.id, duration });
163
+ return sqlBackendWriteMethods.store.call(this, entry);
248
164
  }
249
- /**
250
- * Compare-and-swap update: writes ONLY if the row's stored `version` still
251
- * equals `expectedVersion`. The check and the write are one SQL statement
252
- * (`UPDATE ... WHERE id = ? AND version = ?`), so SQLite's own single-
253
- * statement atomicity — the same guarantee `store()`'s INSERT OR REPLACE
254
- * already relies on — makes this a real compare-and-swap against every
255
- * other writer, in this process or another, with no bridge-level lock.
256
- *
257
- * Exists for memory-KG review finding K5: a getByKey() + store() read-
258
- * merge-write (what every caller did before this) leaves a window in which
259
- * two concurrent callers each merge onto the same row and the second
260
- * store() silently overwrites the first's contribution. A caller that reads
261
- * a row's version, merges onto it, and writes back through here instead
262
- * gets `false` when its merge has gone stale, rather than winning a race it
263
- * did not know it was running.
264
- *
265
- * Only the columns a claim-ledger merge (memory-kg.ts) actually changes are
266
- * updated; embeddings are left untouched, matching storeSync's own rule that
267
- * a revision without a new vector keeps the one it had.
268
- *
269
- * @returns true when the write landed, false when the row's version had
270
- * already moved (or the row no longer exists) — the caller must re-read and
271
- * re-merge, never assume the write happened.
272
- */
273
165
  async storeIfVersion(entry, expectedVersion) {
274
- this.ensureInitialized();
275
- this.validateTags(entry.tags);
276
- const d = this.driver;
277
- return d.transaction(() => {
278
- const changes = d.run(`UPDATE memory_entries
279
- SET content = ?, tags = ?, metadata = ?, updated_at = ?, version = ?, "references" = ?
280
- WHERE id = ? AND version = ?`, [
281
- entry.content,
282
- JSON.stringify(entry.tags),
283
- JSON.stringify(entry.metadata),
284
- entry.updatedAt,
285
- entry.version,
286
- JSON.stringify(entry.references),
287
- entry.id,
288
- expectedVersion,
289
- ]);
290
- if (changes === 0)
291
- return false;
292
- d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [entry.id]);
293
- for (const tag of entry.tags) {
294
- d.run('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)', [
295
- entry.id,
296
- tag,
297
- ]);
298
- }
299
- return true;
300
- });
166
+ return sqlBackendWriteMethods.storeIfVersion.call(this, entry, expectedVersion);
301
167
  }
302
- /**
303
- * Compare-and-swap create: the create-side counterpart to `storeIfVersion`.
304
- * Inserts ONLY if no row already occupies this id or this namespace/key pair
305
- * (the schema's `UNIQUE(namespace, key)` index — see sql-schema.ts).
306
- *
307
- * Exists for the same K5 race on the CREATE side: two concurrent callers
308
- * asserting the same new KG entity mint the same deterministic `key` but a
309
- * different random row `id` (see memory-bridge.ts's upsert). Plain `store()`
310
- * (INSERT OR REPLACE) resolves that collision by silently deleting whichever
311
- * row landed first — exactly the lost update K5 describes, just on first
312
- * write instead of a merge. `INSERT OR IGNORE` instead leaves the first row
313
- * untouched and reports that it did, so the loser re-reads and merges onto
314
- * it instead of clobbering it.
315
- *
316
- * @returns true when this call created the row, false when one already
317
- * existed — the caller must re-read and merge rather than assume it won.
318
- */
319
168
  async storeIfAbsent(entry) {
320
- this.ensureInitialized();
321
- this.validateTags(entry.tags);
322
- const d = this.driver;
323
- return d.transaction(() => {
324
- const changes = d.run(`INSERT OR IGNORE INTO memory_entries (
325
- id, key, content, type, namespace, tags, metadata, owner_id, access_level,
326
- created_at, updated_at, expires_at, event_at, version, "references",
327
- access_count, last_accessed_at
328
- ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, [
329
- entry.id,
330
- entry.key,
331
- entry.content,
332
- entry.type,
333
- entry.namespace,
334
- JSON.stringify(entry.tags),
335
- JSON.stringify(entry.metadata),
336
- entry.ownerId || null,
337
- entry.accessLevel,
338
- entry.createdAt,
339
- entry.updatedAt,
340
- entry.expiresAt || null,
341
- entry.eventAt ?? null,
342
- entry.version,
343
- JSON.stringify(entry.references),
344
- entry.accessCount,
345
- entry.lastAccessedAt,
346
- ]);
347
- if (changes === 0)
348
- return false;
349
- for (const tag of entry.tags) {
350
- d.run('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)', [
351
- entry.id,
352
- tag,
353
- ]);
354
- }
355
- if (entry.embedding && entry.embedding.byteLength > 0) {
356
- const bytes = Buffer.from(entry.embedding.buffer, entry.embedding.byteOffset, entry.embedding.byteLength);
357
- d.run('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)', [
358
- entry.id,
359
- bytes,
360
- ]);
361
- }
362
- return true;
363
- });
364
- }
365
- /** Synchronous store body, shared by store() and bulkInsert(). */
169
+ return sqlBackendWriteMethods.storeIfAbsent.call(this, entry);
170
+ }
366
171
  storeSync(entry) {
367
- const d = this.driver;
368
- // Read any existing embedding BEFORE INSERT OR REPLACE fires the CASCADE
369
- // that would delete it — an entry updated without a vector must keep the
370
- // one it already had.
371
- let embeddingToStore = entry.embedding;
372
- if (!embeddingToStore) {
373
- const existing = d.get('SELECT embedding FROM memory_embeddings WHERE entry_id = ?', [
374
- entry.id,
375
- ]);
376
- const buf = existing?.embedding;
377
- if (buf && buf.byteLength > 0) {
378
- embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
379
- }
380
- }
381
- d.run(`INSERT OR REPLACE INTO memory_entries (
382
- id, key, content, type, namespace, tags, metadata, owner_id, access_level,
383
- created_at, updated_at, expires_at, event_at, version, "references",
384
- access_count, last_accessed_at
385
- ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, [
386
- entry.id,
387
- entry.key,
388
- entry.content,
389
- entry.type,
390
- entry.namespace,
391
- JSON.stringify(entry.tags),
392
- JSON.stringify(entry.metadata),
393
- entry.ownerId || null,
394
- entry.accessLevel,
395
- entry.createdAt,
396
- entry.updatedAt,
397
- entry.expiresAt || null,
398
- entry.eventAt ?? null,
399
- entry.version,
400
- JSON.stringify(entry.references),
401
- entry.accessCount,
402
- entry.lastAccessedAt,
403
- ]);
404
- d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [entry.id]);
405
- for (const tag of entry.tags) {
406
- d.run('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)', [
407
- entry.id,
408
- tag,
409
- ]);
410
- }
411
- if (embeddingToStore) {
412
- // Slice by byteOffset/byteLength: for copies under Node's 4KB pooling
413
- // threshold, `.buffer` refers to the entire shared pool, so writing it
414
- // whole silently stores unrelated memory.
415
- const bytes = Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength);
416
- d.run('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)', [
417
- entry.id,
418
- bytes,
419
- ]);
420
- }
172
+ sqlBackendWriteMethods.storeSync.call(this, entry);
421
173
  }
422
174
  async bulkInsert(entries) {
423
- this.ensureInitialized();
424
- for (const e of entries)
425
- this.validateTags(e.tags);
426
- this.driver?.transaction(() => {
427
- for (const entry of entries)
428
- this.storeSync(entry);
429
- });
430
- this.emit('bulk:inserted', { count: entries.length });
175
+ return sqlBackendWriteMethods.bulkInsert.call(this, entries);
431
176
  }
432
177
  async update(id, update) {
433
- this.ensureInitialized();
434
- const entry = await this.get(id);
435
- if (!entry)
436
- return null;
437
- if (update.content !== undefined)
438
- entry.content = update.content;
439
- if (update.tags !== undefined)
440
- entry.tags = update.tags;
441
- if (update.metadata !== undefined)
442
- entry.metadata = { ...entry.metadata, ...update.metadata };
443
- if (update.accessLevel !== undefined)
444
- entry.accessLevel = update.accessLevel;
445
- if (update.expiresAt !== undefined)
446
- entry.expiresAt = update.expiresAt;
447
- if (update.references !== undefined)
448
- entry.references = update.references;
449
- entry.updatedAt = Date.now();
450
- entry.version++;
451
- await this.store(entry);
452
- this.emit('entry:updated', { id });
453
- return entry;
178
+ return sqlBackendWriteMethods.update.call(this, id, update);
454
179
  }
455
180
  async delete(id) {
456
- this.ensureInitialized();
457
- const d = this.driver;
458
- // Explicit tag cleanup as well as the CASCADE — belt and braces, since FK
459
- // enforcement is a pragma that not every driver honours.
460
- d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [id]);
461
- d.run('DELETE FROM memory_embeddings WHERE entry_id = ?', [id]);
462
- const changes = d.run('DELETE FROM memory_entries WHERE id = ?', [id]);
463
- if (changes > 0) {
464
- this.emit('entry:deleted', { id });
465
- return true;
466
- }
467
- return false;
181
+ return sqlBackendWriteMethods.delete.call(this, id);
468
182
  }
469
183
  async bulkDelete(ids) {
470
- this.ensureInitialized();
471
- const d = this.driver;
472
- const count = d.transaction(() => {
473
- let deleted = 0;
474
- for (const id of ids) {
475
- d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [id]);
476
- d.run('DELETE FROM memory_embeddings WHERE entry_id = ?', [id]);
477
- if (d.run('DELETE FROM memory_entries WHERE id = ?', [id]) > 0)
478
- deleted++;
479
- }
480
- return deleted;
481
- });
482
- this.emit('bulk:deleted', { count });
483
- return count;
184
+ return sqlBackendWriteMethods.bulkDelete.call(this, ids);
484
185
  }
485
186
  async clearNamespace(namespace) {
486
- this.ensureInitialized();
487
- const d = this.driver;
488
- const count = d.transaction(() => {
489
- d.run(`DELETE FROM memory_entry_tags
490
- WHERE entry_id IN (SELECT id FROM memory_entries WHERE namespace = ?)`, [namespace]);
491
- const changes = d.run('DELETE FROM memory_entries WHERE namespace = ?', [namespace]);
492
- d.run('DELETE FROM memory_embeddings WHERE entry_id NOT IN (SELECT id FROM memory_entries)');
493
- return changes;
494
- });
495
- this.emit('namespace:cleared', { namespace, count });
496
- return count;
497
- }
498
- // ===== Reads =============================================================
187
+ return sqlBackendWriteMethods.clearNamespace.call(this, namespace);
188
+ }
189
+ // ===== Reads ==============================================================
190
+ // Bodies live in sql-backend-reads.ts (file-size sweep).
499
191
  async get(id, agentId) {
500
- this.ensureInitialized();
501
- const startTime = performance.now();
502
- const row = this.driver?.get('SELECT memory_entries.*, emb.embedding AS _emb FROM memory_entries LEFT JOIN memory_embeddings emb ON emb.entry_id = memory_entries.id WHERE memory_entries.id = ?', [id]);
503
- if (!row)
504
- return null;
505
- // Collaborative memory promotion — https://arxiv.org/abs/2505.18279
506
- const AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,128}$/;
507
- if (agentId && AGENT_ID_RE.test(agentId)) {
508
- try {
509
- this.driver?.run('INSERT OR IGNORE INTO agent_reads (entry_id, agent_id, read_at) VALUES (?, ?, ?)', [id, agentId, Date.now()]);
510
- this._readCount++;
511
- if (this._readCount % 1000 === 0)
512
- this.checkAndPromoteEntry(id);
513
- }
514
- catch {
515
- /* non-critical */
516
- }
517
- }
518
- const entry = this.rowToEntry(row);
519
- this.emit('entry:retrieved', { id, duration: performance.now() - startTime });
520
- return entry;
192
+ return sqlBackendReadMethods.get.call(this, id, agentId);
521
193
  }
522
- /**
523
- * Promote an entry to 'team' once 3+ distinct agents have read it within 24h.
524
- * https://arxiv.org/abs/2505.18279
525
- */
526
194
  checkAndPromoteEntry(entryId) {
527
- const d = this.driver;
528
- if (!d)
529
- return;
530
- const cutoff = Date.now() - 24 * 60 * 60 * 1000;
531
- d.run('DELETE FROM agent_reads WHERE read_at <= ?', [cutoff]);
532
- const row = d.get('SELECT COUNT(DISTINCT agent_id) as cnt FROM agent_reads WHERE entry_id = ? AND read_at > ?', [entryId, cutoff]);
533
- if (Number(row?.cnt ?? 0) >= 3) {
534
- d.run("UPDATE memory_entries SET access_level = 'team' WHERE id = ? AND access_level = 'private'", [entryId]);
535
- }
195
+ sqlBackendReadMethods.checkAndPromoteEntry.call(this, entryId);
536
196
  }
537
197
  async getByKey(namespace, key) {
538
- this.ensureInitialized();
539
- const startTime = performance.now();
540
- const row = this.driver?.get('SELECT memory_entries.*, emb.embedding AS _emb FROM memory_entries LEFT JOIN memory_embeddings emb ON emb.entry_id = memory_entries.id WHERE memory_entries.namespace = ? AND memory_entries.key = ?', [namespace, key]);
541
- if (!row)
542
- return null;
543
- const entry = this.rowToEntry(row);
544
- this.emit('entry:retrieved', { namespace, key, duration: performance.now() - startTime });
545
- return entry;
198
+ return sqlBackendReadMethods.getByKey.call(this, namespace, key);
546
199
  }
547
200
  async query(query) {
548
- this.ensureInitialized();
549
- const startTime = performance.now();
550
- // PKG-3: LEFT JOIN memory_embeddings once so rowToEntry can read the
551
- // embedding column without an N+1 round-trip per result row.
552
- let sql = 'SELECT memory_entries.*, emb.embedding AS _emb FROM memory_entries LEFT JOIN memory_embeddings emb ON emb.entry_id = memory_entries.id WHERE 1=1';
553
- const params = [];
554
- if (query.namespace) {
555
- sql += ' AND namespace = ?';
556
- params.push(query.namespace);
557
- }
558
- if (query.key) {
559
- sql += ' AND key = ?';
560
- params.push(query.key);
561
- }
562
- if (query.keyPrefix) {
563
- sql += ' AND key LIKE ?';
564
- params.push(`${query.keyPrefix}%`);
565
- }
566
- if (query.memoryType) {
567
- sql += ' AND type = ?';
568
- params.push(query.memoryType);
569
- }
570
- if (query.accessLevel) {
571
- sql += ' AND access_level = ?';
572
- params.push(query.accessLevel);
573
- }
574
- if (query.ownerId) {
575
- sql += ' AND owner_id = ?';
576
- params.push(query.ownerId);
577
- }
578
- if (query.createdAfter) {
579
- sql += ' AND created_at >= ?';
580
- params.push(query.createdAfter);
581
- }
582
- if (query.createdBefore) {
583
- sql += ' AND created_at <= ?';
584
- params.push(query.createdBefore);
585
- }
586
- if (query.updatedAfter) {
587
- sql += ' AND updated_at >= ?';
588
- params.push(query.updatedAfter);
589
- }
590
- if (query.updatedBefore) {
591
- sql += ' AND updated_at <= ?';
592
- params.push(query.updatedBefore);
593
- }
594
- // Bi-temporal event-time filters (arXiv:2501.13956 — Zep/Graphiti)
595
- if (query.eventAfter) {
596
- sql += ' AND event_at >= ?';
597
- params.push(query.eventAfter);
598
- }
599
- if (query.eventBefore) {
600
- sql += ' AND event_at <= ?';
601
- params.push(query.eventBefore);
602
- }
603
- if (!query.includeExpired) {
604
- sql += ' AND (expires_at IS NULL OR expires_at > ?)';
605
- params.push(Date.now());
606
- }
607
- // MemoryQuery.tags is documented as "entries must have all specified tags"
608
- // (types.ts). Counting distinct matches enforces that; an EXISTS(... IN ...)
609
- // would be ANY-match, which is exactly the divergence that made the two old
610
- // backends return different result sets for the same call.
611
- if (query.tags && query.tags.length > 0) {
612
- this.validateTags(query.tags);
613
- const placeholders = query.tags.map(() => '?').join(', ');
614
- sql += ` AND (
615
- SELECT COUNT(DISTINCT t.tag) FROM memory_entry_tags t
616
- WHERE t.entry_id = memory_entries.id AND t.tag IN (${placeholders})
617
- ) = ?`;
618
- params.push(...query.tags, query.tags.length);
619
- }
620
- const colMap = {
621
- createdAt: 'created_at',
622
- updatedAt: 'updated_at',
623
- lastAccessedAt: 'last_accessed_at',
624
- accessCount: 'access_count',
625
- key: 'key',
626
- };
627
- const orderCol = query.sortField && query.sortField !== 'score' && colMap[query.sortField]
628
- ? colMap[query.sortField]
629
- : 'created_at';
630
- const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
631
- sql += ` ORDER BY ${orderCol} ${orderDir} LIMIT ?`;
632
- const effectiveLimit = Math.min(Math.max(1, query.limit ?? MAX_QUERY_LIMIT), MAX_QUERY_LIMIT);
633
- params.push(effectiveLimit);
634
- if (query.offset) {
635
- sql += ' OFFSET ?';
636
- params.push(query.offset);
637
- }
638
- const rows = this.driver?.all(sql, params) ?? [];
639
- const results = rows.map((r) => this.rowToEntry(r));
640
- const duration = performance.now() - startTime;
641
- this.stats.queryCount++;
642
- this.stats.totalQueryTime += duration;
643
- this.emit('query:executed', { query, resultCount: results.length, duration });
644
- return results;
201
+ return sqlBackendReadMethods.query.call(this, query);
645
202
  }
646
- /**
647
- * Semantic search. Below MONOMIND_HNSW_THRESHOLD active embedded entries
648
- * (default 5000), brute-force cosine over stored embeddings — namespace-
649
- * and TTL-filtered in SQL — stays cheaper (a few tens of ms at second-brain
650
- * scale). Above it, getAnnIndex() builds (or loads a persisted) HNSW graph
651
- * and this searches that instead; results are still namespace/threshold
652
- * filtered post-search to match the brute-force semantics exactly.
653
- */
654
203
  async search(embedding, options) {
655
- this.ensureInitialized();
656
- const ns = options.filters?.namespace;
657
- const ann = await this.getAnnIndex(embedding.length).catch(() => null);
658
- if (ann) {
659
- const applyFilters = (raw) => {
660
- const out = [];
661
- for (const r of raw) {
662
- const entry = ann.entries.get(r.id);
663
- if (!entry)
664
- continue;
665
- if (ns && entry.namespace !== ns)
666
- continue;
667
- const score = 1 - r.distance;
668
- if (options.threshold !== undefined && score < options.threshold)
669
- continue;
670
- out.push({ entry, score, distance: r.distance });
671
- if (out.length >= options.k)
672
- break;
673
- }
674
- return out;
675
- };
676
- const overFetch = Math.max(options.k * 4, options.k + 20);
677
- let results = applyFilters(await ann.index.search(embedding, Math.min(overFetch, ann.entries.size)));
678
- // A fixed over-fetch multiple assumes matches are spread roughly evenly
679
- // through the globally-nearest candidates. A namespace filter can
680
- // violate that — a namespace's true nearest neighbors may simply not be
681
- // among the top `overFetch` globally, understating recall (or
682
- // returning nothing) even though matches exist elsewhere in the graph.
683
- // There's no way to know how deep those matches rank without searching
684
- // further, so the only correct fallback is to widen all the way to the
685
- // full index rather than guessing a bigger-but-still-arbitrary number.
686
- if (ns && results.length < options.k && ann.entries.size > overFetch) {
687
- results = applyFilters(await ann.index.search(embedding, ann.entries.size));
688
- }
689
- results.sort((a, b) => b.score - a.score);
690
- return results;
691
- }
692
- const rows = this.driver?.iterate(`SELECT e.*, emb.embedding AS _emb
693
- FROM memory_entries e
694
- JOIN memory_embeddings emb ON emb.entry_id = e.id
695
- WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
696
- ${ns ? 'AND e.namespace = ?' : ''}`, ns ? [Date.now(), ns] : [Date.now()]) ?? [];
697
- const results = [];
698
- for (const row of rows) {
699
- const buf = row._emb;
700
- if (!buf || buf.byteLength % 4 !== 0)
701
- continue;
702
- const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
703
- if (vec.length !== embedding.length)
704
- continue;
705
- const similarity = cosineSimilarity(embedding, vec);
706
- if (options.threshold !== undefined && similarity < options.threshold)
707
- continue;
708
- // PKG-3: row already carries _emb from the JOIN; rowToEntry reads it
709
- // directly instead of re-querying memory_embeddings per row.
710
- results.push({ entry: this.rowToEntry(row), score: similarity, distance: 1 - similarity });
711
- }
712
- results.sort((a, b) => b.score - a.score);
713
- return results.slice(0, options.k);
204
+ return sqlBackendReadMethods.search.call(this, embedding, options);
714
205
  }
715
- /**
716
- * FTS5-accelerated keyword search (Issue #66).
717
- *
718
- * When the FTS5 index is available, text matching runs inside SQLite via
719
- * `MATCH` — orders of magnitude faster than loading 50k rows into JS. When
720
- * FTS5 is unavailable (e.g. sql.js WASM compiled without the extension) the
721
- * method returns `null` so the caller can fall back to JS-side matching.
722
- *
723
- * `queryText` is the raw user query; it is FTS5-tokenized automatically.
724
- * Special characters are escaped to prevent FTS5 syntax errors.
725
- */
726
206
  async keywordSearch(queryText, options = {}) {
727
- this.ensureInitialized();
728
- if (!this._fts5Available)
729
- return null;
730
- const limit = Math.min(Math.max(1, options.limit ?? 50), MAX_QUERY_LIMIT);
731
- // Escape FTS5 special characters and build a query where every token must
732
- // appear (implicit AND). Tokens shorter than 2 chars are dropped — they
733
- // produce noise and FTS5 may reject single-char tokens depending on the
734
- // tokenizer configuration.
735
- const tokens = queryText
736
- .replace(/[":*^~(){}[\]\\]/g, ' ')
737
- .split(/\s+/)
738
- .filter((t) => t.length > 1)
739
- .map((t) => `"${t}"`);
740
- if (!tokens.length)
741
- return null;
742
- const d = this.driver;
743
- const ns = options.namespace;
744
- const search = (match) => d.all(`SELECT f.entry_id, f.key, f.content, e.namespace, rank
745
- FROM memory_entries_fts f
746
- JOIN memory_entries e ON e.id = f.entry_id
747
- WHERE memory_entries_fts MATCH ?
748
- AND (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
749
- ${ns ? 'AND e.namespace = ?' : ''}
750
- ORDER BY rank
751
- LIMIT ?`, ns ? [match, Date.now(), ns, limit] : [match, Date.now(), limit]);
752
- // Preserve the precise all-terms query first. Natural-language queries
753
- // often contain context words absent from the target document, though; a
754
- // zero-result strict search should still surface the best lexical leads.
755
- let rows = search(tokens.join(' '));
756
- if (rows.length === 0 && tokens.length > 1)
757
- rows = search(tokens.join(' OR '));
758
- return rows.map((r) => ({
759
- id: String(r.entry_id),
760
- key: String(r.key),
761
- content: String(r.content),
762
- namespace: String(r.namespace),
763
- // FTS5 rank is negative (lower = better match); invert to a 0–1 score.
764
- rank: Number(r.rank),
765
- }));
207
+ return sqlBackendReadMethods.keywordSearch.call(this, queryText, options);
766
208
  }
767
209
  /** Whether FTS5 full-text search is available on this backend instance. */
768
210
  get fts5Available() {
769
211
  return this._fts5Available;
770
212
  }
771
213
  async count(namespace) {
772
- this.ensureInitialized();
773
- const row = namespace
774
- ? this.driver?.get('SELECT COUNT(*) as count FROM memory_entries WHERE namespace = ?', [
775
- namespace,
776
- ])
777
- : this.driver?.get('SELECT COUNT(*) as count FROM memory_entries');
778
- return Number(row?.count ?? 0);
214
+ return sqlBackendReadMethods.count.call(this, namespace);
779
215
  }
780
216
  async listNamespaces() {
781
- this.ensureInitialized();
782
- return (this.driver?.all('SELECT DISTINCT namespace FROM memory_entries') ?? []).map((r) => String(r.namespace));
217
+ return sqlBackendReadMethods.listNamespaces.call(this);
783
218
  }
784
- // ===== Introspection =====================================================
219
+ // ===== Introspection ======================================================
220
+ // Bodies live in sql-backend-introspect.ts (file-size sweep).
785
221
  async getStats() {
786
- this.ensureInitialized();
787
- const d = this.driver;
788
- const entriesByNamespace = {};
789
- for (const row of d.all('SELECT namespace, COUNT(*) as count FROM memory_entries GROUP BY namespace')) {
790
- entriesByNamespace[String(row.namespace)] = Number(row.count);
791
- }
792
- const entriesByType = {
793
- episodic: 0,
794
- semantic: 0,
795
- working: 0,
796
- cache: 0,
797
- };
798
- for (const row of d.all('SELECT type, COUNT(*) as count FROM memory_entries GROUP BY type')) {
799
- entriesByType[String(row.type)] = Number(row.count);
800
- }
801
- // page_count/page_size are native-only; sql.js reports 0 rather than a
802
- // fabricated figure.
803
- let memoryUsage = 0;
804
- const pageCount = d.pragma('page_count');
805
- const pageSize = d.pragma('page_size');
806
- if (typeof pageCount === 'number' && typeof pageSize === 'number') {
807
- memoryUsage = pageCount * pageSize;
808
- }
809
- return {
810
- totalEntries: await this.count(),
811
- entriesByNamespace,
812
- entriesByType,
813
- memoryUsage,
814
- avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0,
815
- avgSearchTime: 0,
816
- };
222
+ return sqlBackendIntrospectMethods.getStats.call(this);
817
223
  }
818
224
  async healthCheck() {
819
- const issues = [];
820
- const recommendations = [];
821
- if (!this.initialized || !this.driver) {
822
- return {
823
- status: 'unhealthy',
824
- components: {
825
- storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' },
826
- index: { status: 'healthy', latency: 0 },
827
- cache: { status: 'healthy', latency: 0 },
828
- },
829
- timestamp: Date.now(),
830
- issues: ['Backend not initialized'],
831
- recommendations: ['Call initialize() before using'],
832
- };
833
- }
834
- let storageHealth;
835
- try {
836
- const integrity = this.driver.pragma('integrity_check');
837
- if (integrity === undefined || integrity === 'ok') {
838
- // undefined means the driver cannot run the check, not that it failed.
839
- storageHealth = { status: 'healthy', latency: 0 };
840
- }
841
- else {
842
- issues.push('Database integrity check failed');
843
- recommendations.push('Run VACUUM to repair database');
844
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' };
845
- }
846
- }
847
- catch (error) {
848
- issues.push('Failed to check database integrity');
849
- storageHealth = { status: 'unhealthy', latency: 0, message: String(error) };
850
- }
851
- const totalEntries = await this.count();
852
- const utilizationPercent = (totalEntries / this.config.maxEntries) * 100;
853
- if (utilizationPercent > 95) {
854
- issues.push('Storage utilization critical (>95%)');
855
- recommendations.push('Cleanup old data or increase maxEntries');
856
- storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' };
857
- }
858
- else if (utilizationPercent > 80) {
859
- issues.push('Storage utilization high (>80%)');
860
- recommendations.push('Consider cleanup');
861
- if (storageHealth.status === 'healthy') {
862
- storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' };
863
- }
864
- }
865
- const status = storageHealth.status === 'unhealthy'
866
- ? 'unhealthy'
867
- : storageHealth.status === 'degraded'
868
- ? 'degraded'
869
- : 'healthy';
870
- return {
871
- status,
872
- components: {
873
- storage: storageHealth,
874
- index: { status: 'healthy', latency: 0 },
875
- cache: { status: 'healthy', latency: 0 },
876
- },
877
- timestamp: Date.now(),
878
- issues,
879
- recommendations,
880
- };
225
+ return sqlBackendIntrospectMethods.healthCheck.call(this);
881
226
  }
882
227
  /** Flush to durable storage. No-op on write-through drivers. */
883
228
  async persist() {
884
- if (!this.driver)
885
- return;
886
- await this.driver.persist();
229
+ return sqlBackendIntrospectMethods.persist.call(this);
887
230
  }
888
231
  // ===== Internals =========================================================
889
232
  // Every tag write is parameterized, so this is shape sanity rather than