@monoes/memory 1.0.9 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -11
- package/dist/controller-registry.d.ts +7 -5
- package/dist/controller-registry.d.ts.map +1 -1
- package/dist/controller-registry.js +8 -6
- package/dist/controller-registry.js.map +1 -1
- package/dist/index.d.ts +38 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +39 -12
- package/dist/index.js.map +1 -1
- package/dist/migration.d.ts +3 -2
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +3 -2
- package/dist/migration.js.map +1 -1
- package/dist/sql-backend.d.ts +81 -0
- package/dist/sql-backend.d.ts.map +1 -0
- package/dist/sql-backend.js +553 -0
- package/dist/sql-backend.js.map +1 -0
- package/dist/sql-driver.d.ts +135 -0
- package/dist/sql-driver.d.ts.map +1 -0
- package/dist/sql-driver.js +187 -0
- package/dist/sql-driver.js.map +1 -0
- package/dist/sql-schema.d.ts +74 -0
- package/dist/sql-schema.d.ts.map +1 -0
- package/dist/sql-schema.js +178 -0
- package/dist/sql-schema.js.map +1 -0
- package/dist/sqlite-backend.d.ts +16 -105
- package/dist/sqlite-backend.d.ts.map +1 -1
- package/dist/sqlite-backend.js +32 -723
- package/dist/sqlite-backend.js.map +1 -1
- package/dist/sqljs-backend.d.ts +26 -98
- package/dist/sqljs-backend.d.ts.map +1 -1
- package/dist/sqljs-backend.js +75 -609
- package/dist/sqljs-backend.js.map +1 -1
- package/dist/types.d.ts +5 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -2
- package/dist/types.js.map +1 -1
- package/package.json +5 -4
- package/dist/lancedb-backend.d.ts +0 -79
- package/dist/lancedb-backend.d.ts.map +0 -1
- package/dist/lancedb-backend.js +0 -567
- package/dist/lancedb-backend.js.map +0 -1
package/dist/sqlite-backend.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SQLite Memory Backend
|
|
2
|
+
* SQLite Memory Backend (better-sqlite3)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* A thin driver-selecting wrapper around SqlBackend, which holds all the
|
|
5
|
+
* behaviour. This file previously carried a full ~930-line IMemoryBackend
|
|
6
|
+
* implementation that was near-duplicated in sqljs-backend.ts; the two drifted
|
|
7
|
+
* into different schemas and different query semantics. See sql-backend.ts.
|
|
8
|
+
*
|
|
9
|
+
* The class name and config shape are unchanged — this is the published API.
|
|
7
10
|
*
|
|
8
11
|
* @module v1/memory/sqlite-backend
|
|
9
12
|
*/
|
|
10
|
-
import { EventEmitter } from 'node:events';
|
|
11
13
|
import Database from 'better-sqlite3';
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
* Default configuration values
|
|
15
|
-
*/
|
|
14
|
+
import { SqlBackend } from './sql-backend.js';
|
|
15
|
+
import { BetterSqliteDriver } from './sql-driver.js';
|
|
16
16
|
const DEFAULT_CONFIG = {
|
|
17
17
|
databasePath: ':memory:',
|
|
18
18
|
walMode: true,
|
|
@@ -22,726 +22,35 @@ const DEFAULT_CONFIG = {
|
|
|
22
22
|
verbose: false,
|
|
23
23
|
};
|
|
24
24
|
/**
|
|
25
|
-
* SQLite
|
|
25
|
+
* SQLite backend for structured memory storage.
|
|
26
26
|
*
|
|
27
|
-
* Provides
|
|
28
|
-
*
|
|
29
|
-
* - Efficient indexing for exact matches and prefix queries
|
|
30
|
-
* - Full-text search capabilities
|
|
31
|
-
* - Complex SQL queries with joins and aggregations
|
|
32
|
-
* - Persistent storage with WAL mode
|
|
27
|
+
* Provides ACID transactions, indexed lookups, and persistent storage with
|
|
28
|
+
* WAL mode. Behaviour lives in SqlBackend; this class supplies the driver.
|
|
33
29
|
*/
|
|
34
|
-
export class SQLiteBackend extends
|
|
35
|
-
|
|
36
|
-
db = null;
|
|
37
|
-
initialized = false;
|
|
38
|
-
// Performance tracking
|
|
39
|
-
stats = {
|
|
40
|
-
queryCount: 0,
|
|
41
|
-
totalQueryTime: 0,
|
|
42
|
-
writeCount: 0,
|
|
43
|
-
totalWriteTime: 0,
|
|
44
|
-
};
|
|
45
|
-
// Cached prepared statements — prepared once in initialize() to avoid N+1 re-prepare
|
|
46
|
-
stmtGetEmbedding = null;
|
|
47
|
-
stmtInsertEntry = null;
|
|
48
|
-
stmtInsertEmbedding = null;
|
|
49
|
-
stmtDeleteTags = null;
|
|
50
|
-
stmtInsertTag = null;
|
|
51
|
-
// Debounce counter for checkAndPromoteEntry (H2: avoid full-table purge on every get)
|
|
52
|
-
_readCount = 0;
|
|
30
|
+
export class SQLiteBackend extends SqlBackend {
|
|
31
|
+
sqliteConfig;
|
|
53
32
|
constructor(config = {}) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
async initialize() {
|
|
61
|
-
if (this.initialized)
|
|
62
|
-
return;
|
|
63
|
-
// Open database connection
|
|
64
|
-
this.db = new Database(this.config.databasePath, {
|
|
65
|
-
verbose: this.config.verbose ? console.log : undefined,
|
|
33
|
+
const merged = { ...DEFAULT_CONFIG, ...config };
|
|
34
|
+
super({
|
|
35
|
+
defaultNamespace: merged.defaultNamespace,
|
|
36
|
+
embeddingGenerator: merged.embeddingGenerator,
|
|
37
|
+
maxEntries: merged.maxEntries,
|
|
38
|
+
verbose: merged.verbose,
|
|
66
39
|
});
|
|
67
|
-
|
|
68
|
-
if (this.config.walMode) {
|
|
69
|
-
this.db.pragma('journal_mode = WAL');
|
|
70
|
-
}
|
|
71
|
-
// Enable FK enforcement — required for ON DELETE CASCADE to fire
|
|
72
|
-
this.db.pragma('foreign_keys = ON');
|
|
73
|
-
// Performance optimizations
|
|
74
|
-
if (this.config.optimize) {
|
|
75
|
-
this.db.pragma('synchronous = NORMAL');
|
|
76
|
-
this.db.pragma('cache_size = 10000');
|
|
77
|
-
this.db.pragma('temp_store = MEMORY');
|
|
78
|
-
}
|
|
79
|
-
// Create schema
|
|
80
|
-
this.createSchema();
|
|
81
|
-
// Pre-compile hot-path statements to avoid N+1 re-prepare
|
|
82
|
-
this.stmtGetEmbedding = this.db.prepare('SELECT embedding FROM memory_embeddings WHERE entry_id = ?');
|
|
83
|
-
this.stmtInsertEntry = this.db.prepare(`
|
|
84
|
-
INSERT OR REPLACE INTO memory_entries (
|
|
85
|
-
id, key, content, type, namespace, tags, metadata,
|
|
86
|
-
owner_id, access_level, created_at, updated_at, expires_at,
|
|
87
|
-
event_at, version, "references", access_count, last_accessed_at
|
|
88
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
89
|
-
`);
|
|
90
|
-
this.stmtInsertEmbedding = this.db.prepare('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)');
|
|
91
|
-
this.stmtDeleteTags = this.db.prepare('DELETE FROM memory_entry_tags WHERE entry_id = ?');
|
|
92
|
-
this.stmtInsertTag = this.db.prepare('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)');
|
|
93
|
-
this.initialized = true;
|
|
94
|
-
this.emit('initialized');
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Shutdown the backend
|
|
98
|
-
*/
|
|
99
|
-
async shutdown() {
|
|
100
|
-
if (!this.initialized || !this.db)
|
|
101
|
-
return;
|
|
102
|
-
// Optimize database before closing
|
|
103
|
-
if (this.config.optimize) {
|
|
104
|
-
this.db.pragma('optimize');
|
|
105
|
-
}
|
|
106
|
-
this.db.close();
|
|
107
|
-
this.db = null;
|
|
108
|
-
this.stmtGetEmbedding = null;
|
|
109
|
-
this.stmtInsertEntry = null;
|
|
110
|
-
this.stmtInsertEmbedding = null;
|
|
111
|
-
this.stmtDeleteTags = null;
|
|
112
|
-
this.stmtInsertTag = null;
|
|
113
|
-
this.initialized = false;
|
|
114
|
-
this.emit('shutdown');
|
|
40
|
+
this.sqliteConfig = merged;
|
|
115
41
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
async store(entry) {
|
|
120
|
-
this.ensureInitialized();
|
|
121
|
-
const startTime = performance.now();
|
|
122
|
-
this.validateTags(entry.tags);
|
|
123
|
-
const doStore = this.db.transaction((e) => {
|
|
124
|
-
// Read existing embedding BEFORE INSERT OR REPLACE fires CASCADE and deletes it
|
|
125
|
-
let embeddingToStore = e.embedding;
|
|
126
|
-
if (!embeddingToStore) {
|
|
127
|
-
const existingEmb = this.stmtGetEmbedding.get(e.id);
|
|
128
|
-
if (existingEmb?.embedding) {
|
|
129
|
-
const buf = existingEmb.embedding;
|
|
130
|
-
embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
this.stmtInsertEntry.run(e.id, e.key, e.content, e.type, e.namespace, JSON.stringify(e.tags), JSON.stringify(e.metadata), e.ownerId || null, e.accessLevel, e.createdAt, e.updatedAt, e.expiresAt || null, e.eventAt ?? null, e.version, JSON.stringify(e.references), e.accessCount, e.lastAccessedAt);
|
|
134
|
-
this.stmtDeleteTags.run(e.id);
|
|
135
|
-
for (const tag of e.tags) {
|
|
136
|
-
this.stmtInsertTag.run(e.id, tag);
|
|
137
|
-
}
|
|
138
|
-
if (embeddingToStore) {
|
|
139
|
-
this.stmtInsertEmbedding.run(e.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
|
|
140
|
-
}
|
|
42
|
+
async openDriver() {
|
|
43
|
+
const db = new Database(this.sqliteConfig.databasePath, {
|
|
44
|
+
verbose: this.sqliteConfig.verbose ? console.log : undefined,
|
|
141
45
|
});
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
this.
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
*/
|
|
151
|
-
async get(id, agentId) {
|
|
152
|
-
this.ensureInitialized();
|
|
153
|
-
const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
|
|
154
|
-
const row = stmt.get(id);
|
|
155
|
-
if (!row)
|
|
156
|
-
return null;
|
|
157
|
-
// Collaborative memory promotion: track per-agent reads for auto-promotion
|
|
158
|
-
// Source: https://arxiv.org/abs/2505.18279
|
|
159
|
-
const AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,128}$/;
|
|
160
|
-
if (agentId && AGENT_ID_RE.test(agentId)) {
|
|
161
|
-
try {
|
|
162
|
-
this.db.prepare('INSERT OR IGNORE INTO agent_reads (entry_id, agent_id, read_at) VALUES (?, ?, ?)').run(id, agentId, Date.now());
|
|
163
|
-
// Debounce: run full-table purge only every 1000 reads (H2)
|
|
164
|
-
this._readCount++;
|
|
165
|
-
if (this._readCount % 1000 === 0) {
|
|
166
|
-
void this.checkAndPromoteEntry(id);
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
catch { /* non-critical */ }
|
|
170
|
-
}
|
|
171
|
-
return this.rowToEntry(row);
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Auto-promote an entry's access_level to 'team' when 3+ distinct agents
|
|
175
|
-
* have read it within the past 24 hours.
|
|
176
|
-
* Source: https://arxiv.org/abs/2505.18279
|
|
177
|
-
*/
|
|
178
|
-
checkAndPromoteEntry(entryId) {
|
|
179
|
-
if (!this.db)
|
|
180
|
-
return;
|
|
181
|
-
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
|
182
|
-
// Purge stale read records to keep table size bounded
|
|
183
|
-
this.db.prepare('DELETE FROM agent_reads WHERE read_at <= ?').run(cutoff);
|
|
184
|
-
const countRow = this.db.prepare('SELECT COUNT(DISTINCT agent_id) as cnt FROM agent_reads WHERE entry_id = ? AND read_at > ?').get(entryId, cutoff);
|
|
185
|
-
if ((countRow?.cnt ?? 0) >= 3) {
|
|
186
|
-
this.db.prepare("UPDATE memory_entries SET access_level = 'team' WHERE id = ? AND access_level = 'private'").run(entryId);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
/**
|
|
190
|
-
* Get a memory entry by key within a namespace
|
|
191
|
-
*/
|
|
192
|
-
async getByKey(namespace, key) {
|
|
193
|
-
this.ensureInitialized();
|
|
194
|
-
const stmt = this.db.prepare(`
|
|
195
|
-
SELECT * FROM memory_entries
|
|
196
|
-
WHERE namespace = ? AND key = ?
|
|
197
|
-
`);
|
|
198
|
-
const row = stmt.get(namespace, key);
|
|
199
|
-
if (!row)
|
|
200
|
-
return null;
|
|
201
|
-
return this.rowToEntry(row);
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Update a memory entry
|
|
205
|
-
*/
|
|
206
|
-
async update(id, update) {
|
|
207
|
-
this.ensureInitialized();
|
|
208
|
-
const entry = await this.get(id);
|
|
209
|
-
if (!entry)
|
|
210
|
-
return null;
|
|
211
|
-
// Apply updates
|
|
212
|
-
if (update.content !== undefined)
|
|
213
|
-
entry.content = update.content;
|
|
214
|
-
if (update.tags !== undefined)
|
|
215
|
-
entry.tags = update.tags;
|
|
216
|
-
if (update.metadata !== undefined) {
|
|
217
|
-
entry.metadata = { ...entry.metadata, ...update.metadata };
|
|
218
|
-
}
|
|
219
|
-
if (update.accessLevel !== undefined)
|
|
220
|
-
entry.accessLevel = update.accessLevel;
|
|
221
|
-
if (update.expiresAt !== undefined)
|
|
222
|
-
entry.expiresAt = update.expiresAt;
|
|
223
|
-
if (update.references !== undefined)
|
|
224
|
-
entry.references = update.references;
|
|
225
|
-
entry.updatedAt = Date.now();
|
|
226
|
-
entry.version++;
|
|
227
|
-
// Store updated entry
|
|
228
|
-
await this.store(entry);
|
|
229
|
-
this.emit('entry:updated', { id });
|
|
230
|
-
return entry;
|
|
231
|
-
}
|
|
232
|
-
/**
|
|
233
|
-
* Delete a memory entry
|
|
234
|
-
*/
|
|
235
|
-
async delete(id) {
|
|
236
|
-
this.ensureInitialized();
|
|
237
|
-
const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
|
|
238
|
-
// Explicit tag cleanup before entry delete (belt-and-suspenders with CASCADE)
|
|
239
|
-
this.stmtDeleteTags.run(id);
|
|
240
|
-
const result = deleteEntry.run(id);
|
|
241
|
-
if (result.changes > 0) {
|
|
242
|
-
this.emit('entry:deleted', { id });
|
|
243
|
-
return true;
|
|
244
|
-
}
|
|
245
|
-
return false;
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Query memory entries with filters
|
|
249
|
-
*/
|
|
250
|
-
async query(query) {
|
|
251
|
-
this.ensureInitialized();
|
|
252
|
-
const startTime = performance.now();
|
|
253
|
-
let sql = 'SELECT * FROM memory_entries WHERE 1=1';
|
|
254
|
-
const params = [];
|
|
255
|
-
// Build WHERE clauses
|
|
256
|
-
if (query.namespace) {
|
|
257
|
-
sql += ' AND namespace = ?';
|
|
258
|
-
params.push(query.namespace);
|
|
259
|
-
}
|
|
260
|
-
if (query.key) {
|
|
261
|
-
sql += ' AND key = ?';
|
|
262
|
-
params.push(query.key);
|
|
263
|
-
}
|
|
264
|
-
if (query.keyPrefix) {
|
|
265
|
-
sql += ' AND key LIKE ?';
|
|
266
|
-
params.push(`${query.keyPrefix}%`);
|
|
267
|
-
}
|
|
268
|
-
if (query.memoryType) {
|
|
269
|
-
sql += ' AND type = ?';
|
|
270
|
-
params.push(query.memoryType);
|
|
271
|
-
}
|
|
272
|
-
if (query.accessLevel) {
|
|
273
|
-
sql += ' AND access_level = ?';
|
|
274
|
-
params.push(query.accessLevel);
|
|
275
|
-
}
|
|
276
|
-
if (query.ownerId) {
|
|
277
|
-
sql += ' AND owner_id = ?';
|
|
278
|
-
params.push(query.ownerId);
|
|
279
|
-
}
|
|
280
|
-
if (query.createdAfter) {
|
|
281
|
-
sql += ' AND created_at >= ?';
|
|
282
|
-
params.push(query.createdAfter);
|
|
283
|
-
}
|
|
284
|
-
if (query.createdBefore) {
|
|
285
|
-
sql += ' AND created_at <= ?';
|
|
286
|
-
params.push(query.createdBefore);
|
|
287
|
-
}
|
|
288
|
-
if (query.updatedAfter) {
|
|
289
|
-
sql += ' AND updated_at >= ?';
|
|
290
|
-
params.push(query.updatedAfter);
|
|
291
|
-
}
|
|
292
|
-
if (query.updatedBefore) {
|
|
293
|
-
sql += ' AND updated_at <= ?';
|
|
294
|
-
params.push(query.updatedBefore);
|
|
295
|
-
}
|
|
296
|
-
// Bi-temporal event-time filters (arXiv:2501.13956 — Zep/Graphiti)
|
|
297
|
-
if (query.eventAfter) {
|
|
298
|
-
sql += ' AND event_at >= ?';
|
|
299
|
-
params.push(query.eventAfter);
|
|
300
|
-
}
|
|
301
|
-
if (query.eventBefore) {
|
|
302
|
-
sql += ' AND event_at <= ?';
|
|
303
|
-
params.push(query.eventBefore);
|
|
304
|
-
}
|
|
305
|
-
if (!query.includeExpired) {
|
|
306
|
-
sql += ' AND (expires_at IS NULL OR expires_at > ?)';
|
|
307
|
-
params.push(Date.now());
|
|
308
|
-
}
|
|
309
|
-
// Tag filtering — uses indexed memory_entry_tags table (no json_each scan)
|
|
310
|
-
if (query.tags && query.tags.length > 0) {
|
|
311
|
-
this.validateTags(query.tags);
|
|
312
|
-
const tagPlaceholders = query.tags.map(() => '?').join(', ');
|
|
313
|
-
sql += ` AND EXISTS (
|
|
314
|
-
SELECT 1 FROM memory_entry_tags t
|
|
315
|
-
WHERE t.entry_id = memory_entries.id AND t.tag IN (${tagPlaceholders})
|
|
316
|
-
)`;
|
|
317
|
-
params.push(...query.tags);
|
|
318
|
-
}
|
|
319
|
-
// Pagination — always enforce a cap to prevent full-table scans
|
|
320
|
-
const MAX_QUERY_LIMIT = 10_000;
|
|
321
|
-
const effectiveLimit = Math.min(Math.max(1, query.limit ?? MAX_QUERY_LIMIT), MAX_QUERY_LIMIT);
|
|
322
|
-
const colMap = {
|
|
323
|
-
createdAt: 'created_at', updatedAt: 'updated_at',
|
|
324
|
-
lastAccessedAt: 'last_accessed_at', accessCount: 'access_count', key: 'key',
|
|
325
|
-
};
|
|
326
|
-
const orderCol = (query.sortField && query.sortField !== 'score' && colMap[query.sortField])
|
|
327
|
-
? colMap[query.sortField]
|
|
328
|
-
: 'created_at';
|
|
329
|
-
const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
|
|
330
|
-
sql += ` ORDER BY ${orderCol} ${orderDir}`;
|
|
331
|
-
sql += ' LIMIT ?';
|
|
332
|
-
params.push(effectiveLimit);
|
|
333
|
-
if (query.offset) {
|
|
334
|
-
sql += ' OFFSET ?';
|
|
335
|
-
params.push(query.offset);
|
|
336
|
-
}
|
|
337
|
-
const stmt = this.db.prepare(sql);
|
|
338
|
-
const rows = stmt.all(...params);
|
|
339
|
-
const results = rows.map((row) => this.rowToEntry(row));
|
|
340
|
-
const duration = performance.now() - startTime;
|
|
341
|
-
this.stats.queryCount++;
|
|
342
|
-
this.stats.totalQueryTime += duration;
|
|
343
|
-
return results;
|
|
344
|
-
}
|
|
345
|
-
/**
|
|
346
|
-
* Semantic vector search (not optimized for SQLite, returns empty)
|
|
347
|
-
* Use HybridBackend for semantic search with LanceDB
|
|
348
|
-
*/
|
|
349
|
-
async search(embedding, options) {
|
|
350
|
-
this.ensureInitialized();
|
|
351
|
-
// Brute-force cosine over stored embeddings, namespace-scoped and
|
|
352
|
-
// TTL-filtered in SQL. iterate() streams rows one at a time instead of
|
|
353
|
-
// materializing the whole embedding table (.all() held every BLOB in
|
|
354
|
-
// memory at once). At second-brain scale (10^3–10^5 vectors of 384 dims)
|
|
355
|
-
// this is a few tens of ms; swap in the HNSW index if a profiled
|
|
356
|
-
// workload outgrows it.
|
|
357
|
-
const ns = options.filters?.namespace;
|
|
358
|
-
const rows = this.db.prepare(`
|
|
359
|
-
SELECT e.*, emb.embedding AS _emb
|
|
360
|
-
FROM memory_entries e
|
|
361
|
-
JOIN memory_embeddings emb ON emb.entry_id = e.id
|
|
362
|
-
WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
|
|
363
|
-
${ns ? 'AND e.namespace = ?' : ''}
|
|
364
|
-
`).iterate(...(ns ? [Date.now(), ns] : [Date.now()]));
|
|
365
|
-
const results = [];
|
|
366
|
-
for (const row of rows) {
|
|
367
|
-
const buf = row._emb;
|
|
368
|
-
if (!buf || buf.byteLength % 4 !== 0)
|
|
369
|
-
continue;
|
|
370
|
-
const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
371
|
-
if (vec.length !== embedding.length)
|
|
372
|
-
continue;
|
|
373
|
-
const similarity = cosineSimilarity(embedding, vec);
|
|
374
|
-
if (options.threshold !== undefined && similarity < options.threshold)
|
|
375
|
-
continue;
|
|
376
|
-
delete row._emb;
|
|
377
|
-
results.push({ entry: this.rowToEntry(row), score: similarity, distance: 1 - similarity });
|
|
378
|
-
}
|
|
379
|
-
results.sort((a, b) => b.score - a.score);
|
|
380
|
-
return results.slice(0, options.k);
|
|
381
|
-
}
|
|
382
|
-
/**
|
|
383
|
-
* Bulk insert entries
|
|
384
|
-
*/
|
|
385
|
-
async bulkInsert(entries) {
|
|
386
|
-
this.ensureInitialized();
|
|
387
|
-
const transaction = this.db.transaction((entries) => {
|
|
388
|
-
for (const entry of entries) {
|
|
389
|
-
this.storeSync(entry);
|
|
390
|
-
}
|
|
391
|
-
});
|
|
392
|
-
transaction(entries);
|
|
393
|
-
this.emit('bulk:inserted', { count: entries.length });
|
|
394
|
-
}
|
|
395
|
-
/**
|
|
396
|
-
* Bulk delete entries
|
|
397
|
-
*/
|
|
398
|
-
async bulkDelete(ids) {
|
|
399
|
-
this.ensureInitialized();
|
|
400
|
-
const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
|
|
401
|
-
const transaction = this.db.transaction((ids) => {
|
|
402
|
-
let deleted = 0;
|
|
403
|
-
for (const id of ids) {
|
|
404
|
-
this.stmtDeleteTags.run(id);
|
|
405
|
-
const result = deleteEntry.run(id);
|
|
406
|
-
if (result.changes > 0)
|
|
407
|
-
deleted++;
|
|
408
|
-
}
|
|
409
|
-
return deleted;
|
|
410
|
-
});
|
|
411
|
-
return transaction(ids);
|
|
412
|
-
}
|
|
413
|
-
/**
|
|
414
|
-
* Get entry count
|
|
415
|
-
*/
|
|
416
|
-
async count(namespace) {
|
|
417
|
-
this.ensureInitialized();
|
|
418
|
-
let sql = 'SELECT COUNT(*) as count FROM memory_entries';
|
|
419
|
-
const params = [];
|
|
420
|
-
if (namespace) {
|
|
421
|
-
sql += ' WHERE namespace = ?';
|
|
422
|
-
params.push(namespace);
|
|
423
|
-
}
|
|
424
|
-
const stmt = this.db.prepare(sql);
|
|
425
|
-
const result = stmt.get(...params);
|
|
426
|
-
return result.count;
|
|
427
|
-
}
|
|
428
|
-
/**
|
|
429
|
-
* List all namespaces
|
|
430
|
-
*/
|
|
431
|
-
async listNamespaces() {
|
|
432
|
-
this.ensureInitialized();
|
|
433
|
-
const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
|
|
434
|
-
const rows = stmt.all();
|
|
435
|
-
return rows.map((row) => row.namespace);
|
|
436
|
-
}
|
|
437
|
-
/**
|
|
438
|
-
* Clear all entries in a namespace
|
|
439
|
-
*/
|
|
440
|
-
async clearNamespace(namespace) {
|
|
441
|
-
this.ensureInitialized();
|
|
442
|
-
const deleteTags = this.db.prepare(`
|
|
443
|
-
DELETE FROM memory_entry_tags
|
|
444
|
-
WHERE entry_id IN (SELECT id FROM memory_entries WHERE namespace = ?)
|
|
445
|
-
`);
|
|
446
|
-
const deleteEntries = this.db.prepare('DELETE FROM memory_entries WHERE namespace = ?');
|
|
447
|
-
const deleteOrphanEmbeddings = this.db.prepare(`
|
|
448
|
-
DELETE FROM memory_embeddings
|
|
449
|
-
WHERE entry_id NOT IN (SELECT id FROM memory_entries)
|
|
450
|
-
`);
|
|
451
|
-
let changes = 0;
|
|
452
|
-
this.db.transaction(() => {
|
|
453
|
-
deleteTags.run(namespace);
|
|
454
|
-
changes = deleteEntries.run(namespace).changes;
|
|
455
|
-
deleteOrphanEmbeddings.run();
|
|
456
|
-
})();
|
|
457
|
-
return changes;
|
|
458
|
-
}
|
|
459
|
-
/**
|
|
460
|
-
* Get backend statistics
|
|
461
|
-
*/
|
|
462
|
-
async getStats() {
|
|
463
|
-
this.ensureInitialized();
|
|
464
|
-
// Count by namespace
|
|
465
|
-
const namespaceStmt = this.db.prepare(`
|
|
466
|
-
SELECT namespace, COUNT(*) as count
|
|
467
|
-
FROM memory_entries
|
|
468
|
-
GROUP BY namespace
|
|
469
|
-
`);
|
|
470
|
-
const namespaceRows = namespaceStmt.all();
|
|
471
|
-
const entriesByNamespace = {};
|
|
472
|
-
for (const row of namespaceRows) {
|
|
473
|
-
entriesByNamespace[row.namespace] = row.count;
|
|
474
|
-
}
|
|
475
|
-
// Count by type
|
|
476
|
-
const typeStmt = this.db.prepare(`
|
|
477
|
-
SELECT type, COUNT(*) as count
|
|
478
|
-
FROM memory_entries
|
|
479
|
-
GROUP BY type
|
|
480
|
-
`);
|
|
481
|
-
const typeRows = typeStmt.all();
|
|
482
|
-
const entriesByType = {
|
|
483
|
-
episodic: 0,
|
|
484
|
-
semantic: 0,
|
|
485
|
-
working: 0,
|
|
486
|
-
cache: 0,
|
|
487
|
-
};
|
|
488
|
-
for (const row of typeRows) {
|
|
489
|
-
entriesByType[row.type] = row.count;
|
|
490
|
-
}
|
|
491
|
-
// Get database size
|
|
492
|
-
const pageCount = this.db.pragma('page_count', { simple: true });
|
|
493
|
-
const pageSize = this.db.pragma('page_size', { simple: true });
|
|
494
|
-
const memoryUsage = pageCount * pageSize;
|
|
495
|
-
const totalEntries = await this.count();
|
|
496
|
-
return {
|
|
497
|
-
totalEntries,
|
|
498
|
-
entriesByNamespace,
|
|
499
|
-
entriesByType,
|
|
500
|
-
memoryUsage,
|
|
501
|
-
avgQueryTime: this.stats.queryCount > 0
|
|
502
|
-
? this.stats.totalQueryTime / this.stats.queryCount
|
|
503
|
-
: 0,
|
|
504
|
-
avgSearchTime: 0, // Not applicable for SQLite
|
|
505
|
-
};
|
|
506
|
-
}
|
|
507
|
-
/**
|
|
508
|
-
* Perform health check
|
|
509
|
-
*/
|
|
510
|
-
async healthCheck() {
|
|
511
|
-
const issues = [];
|
|
512
|
-
const recommendations = [];
|
|
513
|
-
if (!this.initialized || !this.db) {
|
|
514
|
-
return {
|
|
515
|
-
status: 'unhealthy',
|
|
516
|
-
components: {
|
|
517
|
-
storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' },
|
|
518
|
-
index: { status: 'healthy', latency: 0 },
|
|
519
|
-
cache: { status: 'healthy', latency: 0 },
|
|
520
|
-
},
|
|
521
|
-
timestamp: Date.now(),
|
|
522
|
-
issues: ['Backend not initialized'],
|
|
523
|
-
recommendations: ['Call initialize() before using'],
|
|
524
|
-
};
|
|
525
|
-
}
|
|
526
|
-
// Check database integrity
|
|
527
|
-
let storageHealth;
|
|
528
|
-
try {
|
|
529
|
-
const integrityCheck = this.db.pragma('integrity_check', { simple: true });
|
|
530
|
-
if (integrityCheck === 'ok') {
|
|
531
|
-
storageHealth = { status: 'healthy', latency: 0 };
|
|
532
|
-
}
|
|
533
|
-
else {
|
|
534
|
-
issues.push('Database integrity check failed');
|
|
535
|
-
recommendations.push('Run VACUUM to repair database');
|
|
536
|
-
storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' };
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
catch (error) {
|
|
540
|
-
issues.push('Failed to check database integrity');
|
|
541
|
-
storageHealth = { status: 'unhealthy', latency: 0, message: String(error) };
|
|
542
|
-
}
|
|
543
|
-
// Check utilization
|
|
544
|
-
const totalEntries = await this.count();
|
|
545
|
-
const utilizationPercent = (totalEntries / this.config.maxEntries) * 100;
|
|
546
|
-
if (utilizationPercent > 95) {
|
|
547
|
-
issues.push('Storage utilization critical (>95%)');
|
|
548
|
-
recommendations.push('Cleanup old data or increase maxEntries');
|
|
549
|
-
storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' };
|
|
550
|
-
}
|
|
551
|
-
else if (utilizationPercent > 80) {
|
|
552
|
-
issues.push('Storage utilization high (>80%)');
|
|
553
|
-
recommendations.push('Consider cleanup');
|
|
554
|
-
if (storageHealth.status === 'healthy') {
|
|
555
|
-
storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' };
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
const status = storageHealth.status === 'unhealthy'
|
|
559
|
-
? 'unhealthy'
|
|
560
|
-
: storageHealth.status === 'degraded'
|
|
561
|
-
? 'degraded'
|
|
562
|
-
: 'healthy';
|
|
563
|
-
return {
|
|
564
|
-
status,
|
|
565
|
-
components: {
|
|
566
|
-
storage: storageHealth,
|
|
567
|
-
index: { status: 'healthy', latency: 0 },
|
|
568
|
-
cache: { status: 'healthy', latency: 0 },
|
|
569
|
-
},
|
|
570
|
-
timestamp: Date.now(),
|
|
571
|
-
issues,
|
|
572
|
-
recommendations,
|
|
573
|
-
};
|
|
574
|
-
}
|
|
575
|
-
// ===== Private Methods =====
|
|
576
|
-
// `/`, `~`, and spaces admitted: `src:<absolute path>` provenance tags are a
|
|
577
|
-
// first-class part of the knowledge pipeline, and every tag write below is
|
|
578
|
-
// parameterized — the regex is shape sanity, not injection defense. The
|
|
579
|
-
// sql.js backend has never validated tags at all, so the strict form also
|
|
580
|
-
// made store success silently depend on which backend loaded.
|
|
581
|
-
static TAG_RE = /^[a-zA-Z0-9_\-.:/~ ]+$/;
|
|
582
|
-
static MAX_TAG_LEN = 512;
|
|
583
|
-
validateTags(tags) {
|
|
584
|
-
for (const tag of tags) {
|
|
585
|
-
if (typeof tag !== 'string' || tag.length > SQLiteBackend.MAX_TAG_LEN || !SQLiteBackend.TAG_RE.test(tag)) {
|
|
586
|
-
throw new Error(`Invalid tag format: "${String(tag).slice(0, 80)}"`);
|
|
587
|
-
}
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
ensureInitialized() {
|
|
591
|
-
if (!this.initialized || !this.db) {
|
|
592
|
-
throw new Error('SQLiteBackend not initialized. Call initialize() first.');
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
createSchema() {
|
|
596
|
-
if (!this.db)
|
|
597
|
-
return;
|
|
598
|
-
// Main entries table
|
|
599
|
-
this.db.exec(`
|
|
600
|
-
CREATE TABLE IF NOT EXISTS memory_entries (
|
|
601
|
-
id TEXT PRIMARY KEY,
|
|
602
|
-
key TEXT NOT NULL,
|
|
603
|
-
content TEXT NOT NULL,
|
|
604
|
-
type TEXT NOT NULL,
|
|
605
|
-
namespace TEXT NOT NULL,
|
|
606
|
-
tags TEXT NOT NULL,
|
|
607
|
-
metadata TEXT NOT NULL,
|
|
608
|
-
owner_id TEXT,
|
|
609
|
-
access_level TEXT NOT NULL,
|
|
610
|
-
created_at INTEGER NOT NULL,
|
|
611
|
-
updated_at INTEGER NOT NULL,
|
|
612
|
-
expires_at INTEGER,
|
|
613
|
-
event_at INTEGER,
|
|
614
|
-
version INTEGER NOT NULL,
|
|
615
|
-
"references" TEXT NOT NULL,
|
|
616
|
-
access_count INTEGER NOT NULL,
|
|
617
|
-
last_accessed_at INTEGER NOT NULL
|
|
618
|
-
);
|
|
619
|
-
|
|
620
|
-
CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace);
|
|
621
|
-
CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key);
|
|
622
|
-
CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type);
|
|
623
|
-
CREATE INDEX IF NOT EXISTS idx_owner_id ON memory_entries(owner_id);
|
|
624
|
-
CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at);
|
|
625
|
-
CREATE INDEX IF NOT EXISTS idx_updated_at ON memory_entries(updated_at);
|
|
626
|
-
CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at);
|
|
627
|
-
|
|
628
|
-
CREATE TABLE IF NOT EXISTS memory_embeddings (
|
|
629
|
-
entry_id TEXT PRIMARY KEY,
|
|
630
|
-
embedding BLOB,
|
|
631
|
-
FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
|
|
632
|
-
);
|
|
633
|
-
`);
|
|
634
|
-
// UNIQUE(namespace,key): the sql.js schema has always enforced it; this
|
|
635
|
-
// backend only had a plain index, so repeated stores under the same key
|
|
636
|
-
// accumulated duplicates and getByKey returned an arbitrary one. Existing
|
|
637
|
-
// databases may already hold duplicates — dedupe (keep newest) before the
|
|
638
|
-
// unique index can be created, then upgrade the index.
|
|
639
|
-
try {
|
|
640
|
-
this.db.exec(`
|
|
641
|
-
DELETE FROM memory_entries WHERE id NOT IN (
|
|
642
|
-
SELECT id FROM (
|
|
643
|
-
SELECT id, ROW_NUMBER() OVER (PARTITION BY namespace, key ORDER BY updated_at DESC, id DESC) AS rn
|
|
644
|
-
FROM memory_entries
|
|
645
|
-
) WHERE rn = 1
|
|
646
|
-
);
|
|
647
|
-
DROP INDEX IF EXISTS idx_namespace_key;
|
|
648
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key_unique ON memory_entries(namespace, key);
|
|
649
|
-
`);
|
|
650
|
-
}
|
|
651
|
-
catch {
|
|
652
|
-
// Very old SQLite without window functions: keep the plain index rather
|
|
653
|
-
// than failing initialization — dup accumulation is the lesser evil.
|
|
654
|
-
this.db.exec('CREATE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key);');
|
|
655
|
-
}
|
|
656
|
-
this.db.exec(`
|
|
657
|
-
|
|
658
|
-
CREATE TABLE IF NOT EXISTS agent_reads (
|
|
659
|
-
entry_id TEXT NOT NULL,
|
|
660
|
-
agent_id TEXT NOT NULL,
|
|
661
|
-
read_at INTEGER NOT NULL,
|
|
662
|
-
UNIQUE(entry_id, agent_id)
|
|
663
|
-
);
|
|
664
|
-
CREATE INDEX IF NOT EXISTS idx_agent_reads_entry ON agent_reads(entry_id);
|
|
665
|
-
CREATE INDEX IF NOT EXISTS idx_agent_reads_at ON agent_reads(read_at);
|
|
666
|
-
|
|
667
|
-
CREATE TABLE IF NOT EXISTS memory_entry_tags (
|
|
668
|
-
entry_id TEXT NOT NULL,
|
|
669
|
-
tag TEXT NOT NULL,
|
|
670
|
-
PRIMARY KEY (entry_id, tag),
|
|
671
|
-
FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE
|
|
672
|
-
);
|
|
673
|
-
CREATE INDEX IF NOT EXISTS idx_entry_tag ON memory_entry_tags(tag, entry_id);
|
|
674
|
-
`);
|
|
675
|
-
// One-time backfill gated by user_version — wrapped in a transaction to hold
|
|
676
|
-
// a write lock and prevent double-backfill under concurrent process startup.
|
|
677
|
-
this.db.transaction(() => {
|
|
678
|
-
const schemaVersion = this.db.pragma('user_version', { simple: true });
|
|
679
|
-
if (schemaVersion < 1) {
|
|
680
|
-
this.db.exec(`
|
|
681
|
-
INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag)
|
|
682
|
-
SELECT memory_entries.id, t.value
|
|
683
|
-
FROM memory_entries, json_each(tags) AS t
|
|
684
|
-
WHERE json_valid(tags) AND t.value IS NOT NULL AND t.value != '';
|
|
685
|
-
`);
|
|
686
|
-
this.db.pragma('user_version = 1');
|
|
687
|
-
}
|
|
688
|
-
})();
|
|
689
|
-
}
|
|
690
|
-
rowToEntry(row) {
|
|
691
|
-
// Get embedding if exists — use pre-compiled statement to avoid N+1 re-prepare
|
|
692
|
-
let embedding;
|
|
693
|
-
const embeddingRow = this.stmtGetEmbedding.get(row.id);
|
|
694
|
-
if (embeddingRow && embeddingRow.embedding) {
|
|
695
|
-
// Slice only the intended byte range — Buffer.from(existingBuffer) for copies
|
|
696
|
-
// under Node's 8KB pool threshold returns a view into the SHARED pool
|
|
697
|
-
// ArrayBuffer at a nonzero offset, so `.buffer` alone spans unrelated pool
|
|
698
|
-
// memory (silent data corruption). byteOffset/byteLength scope it correctly.
|
|
699
|
-
const buf = embeddingRow.embedding;
|
|
700
|
-
embedding = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
701
|
-
}
|
|
702
|
-
return {
|
|
703
|
-
id: row.id,
|
|
704
|
-
key: row.key,
|
|
705
|
-
content: row.content,
|
|
706
|
-
embedding,
|
|
707
|
-
type: row.type,
|
|
708
|
-
namespace: row.namespace,
|
|
709
|
-
tags: JSON.parse(row.tags),
|
|
710
|
-
metadata: JSON.parse(row.metadata),
|
|
711
|
-
ownerId: row.owner_id,
|
|
712
|
-
accessLevel: row.access_level,
|
|
713
|
-
createdAt: row.created_at,
|
|
714
|
-
updatedAt: row.updated_at,
|
|
715
|
-
expiresAt: row.expires_at,
|
|
716
|
-
eventAt: row.event_at ?? undefined,
|
|
717
|
-
version: row.version,
|
|
718
|
-
references: JSON.parse(row.references),
|
|
719
|
-
accessCount: row.access_count,
|
|
720
|
-
lastAccessedAt: row.last_accessed_at,
|
|
721
|
-
};
|
|
722
|
-
}
|
|
723
|
-
/**
|
|
724
|
-
* Synchronous store for use in transactions
|
|
725
|
-
*/
|
|
726
|
-
storeSync(entry) {
|
|
727
|
-
this.validateTags(entry.tags);
|
|
728
|
-
// Read existing embedding before INSERT OR REPLACE cascades-deletes it
|
|
729
|
-
let embeddingToStore = entry.embedding;
|
|
730
|
-
if (!embeddingToStore) {
|
|
731
|
-
const existingEmb = this.stmtGetEmbedding.get(entry.id);
|
|
732
|
-
if (existingEmb?.embedding) {
|
|
733
|
-
const buf = Buffer.from(existingEmb.embedding);
|
|
734
|
-
embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
this.stmtInsertEntry.run(entry.id, entry.key, entry.content, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.eventAt ?? null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt);
|
|
738
|
-
this.stmtDeleteTags.run(entry.id);
|
|
739
|
-
for (const tag of entry.tags) {
|
|
740
|
-
this.stmtInsertTag.run(entry.id, tag);
|
|
741
|
-
}
|
|
742
|
-
if (embeddingToStore) {
|
|
743
|
-
this.stmtInsertEmbedding.run(entry.id, Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength));
|
|
744
|
-
}
|
|
46
|
+
if (this.sqliteConfig.walMode)
|
|
47
|
+
db.pragma('journal_mode = WAL');
|
|
48
|
+
if (this.sqliteConfig.optimize) {
|
|
49
|
+
db.pragma('synchronous = NORMAL');
|
|
50
|
+
db.pragma('cache_size = 10000');
|
|
51
|
+
db.pragma('temp_store = MEMORY');
|
|
52
|
+
}
|
|
53
|
+
return new BetterSqliteDriver(db);
|
|
745
54
|
}
|
|
746
55
|
}
|
|
747
56
|
export default SQLiteBackend;
|