@monoes/memory 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -11
- package/dist/controller-registry.d.ts +7 -5
- package/dist/controller-registry.d.ts.map +1 -1
- package/dist/controller-registry.js +8 -6
- package/dist/controller-registry.js.map +1 -1
- package/dist/index.d.ts +38 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +39 -12
- package/dist/index.js.map +1 -1
- package/dist/migration.d.ts +3 -2
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +3 -2
- package/dist/migration.js.map +1 -1
- package/dist/sql-backend.d.ts +81 -0
- package/dist/sql-backend.d.ts.map +1 -0
- package/dist/sql-backend.js +553 -0
- package/dist/sql-backend.js.map +1 -0
- package/dist/sql-driver.d.ts +135 -0
- package/dist/sql-driver.d.ts.map +1 -0
- package/dist/sql-driver.js +187 -0
- package/dist/sql-driver.js.map +1 -0
- package/dist/sql-schema.d.ts +74 -0
- package/dist/sql-schema.d.ts.map +1 -0
- package/dist/sql-schema.js +178 -0
- package/dist/sql-schema.js.map +1 -0
- package/dist/sqlite-backend.d.ts +16 -106
- package/dist/sqlite-backend.d.ts.map +1 -1
- package/dist/sqlite-backend.js +53 -725
- package/dist/sqlite-backend.js.map +1 -1
- package/dist/sqljs-backend.d.ts +26 -98
- package/dist/sqljs-backend.d.ts.map +1 -1
- package/dist/sqljs-backend.js +75 -609
- package/dist/sqljs-backend.js.map +1 -1
- package/dist/types.d.ts +5 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -2
- package/dist/types.js.map +1 -1
- package/package.json +5 -4
- package/dist/lancedb-backend.d.ts +0 -79
- package/dist/lancedb-backend.d.ts.map +0 -1
- package/dist/lancedb-backend.js +0 -567
- package/dist/lancedb-backend.js.map +0 -1
package/dist/sqljs-backend.js
CHANGED
|
@@ -1,25 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* sql.js (WASM) Memory Backend
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* A thin driver-selecting wrapper around SqlBackend, which holds all the
|
|
5
|
+
* behaviour. This file previously carried its own ~845-line IMemoryBackend
|
|
6
|
+
* implementation parallel to sqlite-backend.ts; the two drifted into different
|
|
7
|
+
* schemas and different query semantics (tag filtering was ANY-match on one and
|
|
8
|
+
* ALL-match on the other for the same call). See sql-backend.ts.
|
|
9
|
+
*
|
|
10
|
+
* Two things remain genuinely specific to this driver and live here:
|
|
11
|
+
* - WASM loading, preferring the copy in node_modules over the CDN.
|
|
12
|
+
* - Durability. sql.js is in-memory, so the database must be exported to
|
|
13
|
+
* disk explicitly — on an interval and at shutdown.
|
|
14
|
+
*
|
|
15
|
+
* Databases written by the pre-unification version stored embeddings inline on
|
|
16
|
+
* memory_entries. They are migrated into memory_embeddings on first open; the
|
|
17
|
+
* migration copies rather than moves and is idempotent (see sql-schema.ts).
|
|
18
|
+
*
|
|
19
|
+
* The class name and config shape are unchanged — this is the published API.
|
|
6
20
|
*
|
|
7
21
|
* @module v1/memory/sqljs-backend
|
|
8
22
|
*/
|
|
9
|
-
import { EventEmitter } from 'node:events';
|
|
10
|
-
import { readFileSync, existsSync } from 'node:fs';
|
|
11
|
-
import { createRequire } from 'node:module';
|
|
12
23
|
import initSqlJs from 'sql.js';
|
|
24
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
25
|
+
import { createRequire } from 'node:module';
|
|
26
|
+
import { SqlBackend } from './sql-backend.js';
|
|
27
|
+
import { SqlJsDriver } from './sql-driver.js';
|
|
13
28
|
import { writeFileAtomicSync } from './atomic-file.js';
|
|
14
|
-
import { cosineSimilarity } from './math-utils.js';
|
|
15
29
|
/**
|
|
16
|
-
* Resolve the
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* Returns null if local resolution fails,
|
|
22
|
-
* the CDN as a last resort.
|
|
30
|
+
* Resolve the WASM binary shipped inside node_modules.
|
|
31
|
+
*
|
|
32
|
+
* Fetching from the sql.js CDN is a silent network dependency in what should be
|
|
33
|
+
* a normal npm install, and is undesirable in offline/air-gapped environments —
|
|
34
|
+
* the same environments most likely to need this fallback, since it only kicks
|
|
35
|
+
* in when native better-sqlite3 fails. Returns null if local resolution fails,
|
|
36
|
+
* so the caller can fall back to the CDN as a last resort.
|
|
23
37
|
*/
|
|
24
38
|
function resolveLocalWasmPath() {
|
|
25
39
|
try {
|
|
@@ -30,622 +44,74 @@ function resolveLocalWasmPath() {
|
|
|
30
44
|
return null;
|
|
31
45
|
}
|
|
32
46
|
}
|
|
33
|
-
/**
|
|
34
|
-
* Default configuration values
|
|
35
|
-
*/
|
|
36
47
|
const DEFAULT_CONFIG = {
|
|
37
48
|
databasePath: ':memory:',
|
|
38
49
|
optimize: true,
|
|
39
50
|
defaultNamespace: 'default',
|
|
40
51
|
maxEntries: 1000000,
|
|
41
52
|
verbose: false,
|
|
42
|
-
autoPersistInterval: 5000,
|
|
53
|
+
autoPersistInterval: 5000,
|
|
43
54
|
};
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
*
|
|
47
|
-
* Provides:
|
|
48
|
-
* - Pure JavaScript/WASM implementation (no native compilation)
|
|
49
|
-
* - Windows, macOS, Linux compatibility
|
|
50
|
-
* - Same SQL interface as better-sqlite3
|
|
51
|
-
* - In-memory with periodic disk persistence
|
|
52
|
-
* - Fallback when native SQLite fails
|
|
53
|
-
*/
|
|
54
|
-
export class SqlJsBackend extends EventEmitter {
|
|
55
|
-
config;
|
|
56
|
-
db = null;
|
|
57
|
-
initialized = false;
|
|
55
|
+
export class SqlJsBackend extends SqlBackend {
|
|
56
|
+
sqlJsConfig;
|
|
58
57
|
persistTimer = null;
|
|
59
|
-
SQL = null;
|
|
60
|
-
// Performance tracking
|
|
61
|
-
stats = {
|
|
62
|
-
queryCount: 0,
|
|
63
|
-
totalQueryTime: 0,
|
|
64
|
-
writeCount: 0,
|
|
65
|
-
totalWriteTime: 0,
|
|
66
|
-
};
|
|
67
58
|
constructor(config = {}) {
|
|
68
|
-
|
|
69
|
-
|
|
59
|
+
const merged = { ...DEFAULT_CONFIG, ...config };
|
|
60
|
+
super({
|
|
61
|
+
defaultNamespace: merged.defaultNamespace,
|
|
62
|
+
embeddingGenerator: merged.embeddingGenerator,
|
|
63
|
+
maxEntries: merged.maxEntries,
|
|
64
|
+
verbose: merged.verbose,
|
|
65
|
+
});
|
|
66
|
+
this.sqlJsConfig = merged;
|
|
70
67
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if (this.initialized)
|
|
76
|
-
return;
|
|
77
|
-
// Load sql.js WASM — prefer an explicit wasmPath, then the copy shipped
|
|
78
|
-
// in node_modules/sql.js/dist, and only fall back to the sql.js CDN if
|
|
79
|
-
// local resolution genuinely fails (e.g. a deployment that strips
|
|
80
|
-
// node_modules and relies on CDN delivery).
|
|
81
|
-
const localWasmPath = this.config.wasmPath ?? resolveLocalWasmPath() ?? undefined;
|
|
82
|
-
this.SQL = await initSqlJs({
|
|
83
|
-
locateFile: localWasmPath
|
|
84
|
-
? () => localWasmPath
|
|
85
|
-
: (file) => `https://sql.js.org/dist/${file}`,
|
|
68
|
+
async openDriver() {
|
|
69
|
+
const localWasmPath = this.sqlJsConfig.wasmPath ?? resolveLocalWasmPath() ?? undefined;
|
|
70
|
+
const SQL = await initSqlJs({
|
|
71
|
+
locateFile: localWasmPath ? () => localWasmPath : (file) => `https://sql.js.org/dist/${file}`,
|
|
86
72
|
});
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
73
|
+
const path = this.sqlJsConfig.databasePath;
|
|
74
|
+
const isMemory = path === ':memory:';
|
|
75
|
+
const db = !isMemory && existsSync(path)
|
|
76
|
+
? new SQL.Database(new Uint8Array(readFileSync(path)))
|
|
77
|
+
: new SQL.Database();
|
|
78
|
+
if (this.sqlJsConfig.verbose) {
|
|
79
|
+
console.log(!isMemory && existsSync(path)
|
|
80
|
+
? `[SqlJsBackend] Loaded database from ${path}`
|
|
81
|
+
: '[SqlJsBackend] Created new in-memory database');
|
|
82
|
+
}
|
|
83
|
+
// Only give the driver a flush function when there is somewhere to flush to.
|
|
84
|
+
const flush = isMemory
|
|
85
|
+
? undefined
|
|
86
|
+
: async (bytes) => {
|
|
87
|
+
const buffer = Buffer.from(bytes);
|
|
88
|
+
writeFileAtomicSync(path, buffer);
|
|
89
|
+
if (this.sqlJsConfig.verbose) {
|
|
90
|
+
console.log(`[SqlJsBackend] Persisted ${buffer.length} bytes to ${path}`);
|
|
91
|
+
}
|
|
92
|
+
this.emit('persisted', { size: buffer.length, path });
|
|
93
|
+
};
|
|
94
|
+
return new SqlJsDriver(db, flush);
|
|
95
|
+
}
|
|
96
|
+
async initialize() {
|
|
97
|
+
await super.initialize();
|
|
98
|
+
if (this.sqlJsConfig.autoPersistInterval > 0 && this.sqlJsConfig.databasePath !== ':memory:') {
|
|
106
99
|
this.persistTimer = setInterval(() => {
|
|
107
|
-
this.persist().catch((
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
}, this.config.autoPersistInterval);
|
|
100
|
+
this.persist().catch((error) => this.emit('error', { operation: 'auto-persist', error }));
|
|
101
|
+
}, this.sqlJsConfig.autoPersistInterval);
|
|
102
|
+
// Do not hold the event loop open purely for periodic persistence.
|
|
111
103
|
if (this.persistTimer.unref)
|
|
112
104
|
this.persistTimer.unref();
|
|
113
105
|
}
|
|
114
|
-
this.initialized = true;
|
|
115
|
-
this.emit('initialized');
|
|
116
106
|
}
|
|
117
|
-
/**
|
|
118
|
-
* Shutdown the backend
|
|
119
|
-
*/
|
|
120
107
|
async shutdown() {
|
|
121
|
-
if (!this.initialized || !this.db)
|
|
122
|
-
return;
|
|
123
|
-
// Stop auto-persist timer
|
|
124
108
|
if (this.persistTimer) {
|
|
125
109
|
clearInterval(this.persistTimer);
|
|
126
110
|
this.persistTimer = null;
|
|
127
111
|
}
|
|
128
|
-
//
|
|
129
|
-
|
|
130
|
-
await this.persist();
|
|
131
|
-
}
|
|
132
|
-
this.db.close();
|
|
133
|
-
this.db = null;
|
|
134
|
-
this.initialized = false;
|
|
135
|
-
this.emit('shutdown');
|
|
136
|
-
}
|
|
137
|
-
/**
|
|
138
|
-
* Create database schema
|
|
139
|
-
*/
|
|
140
|
-
createSchema() {
|
|
141
|
-
if (!this.db)
|
|
142
|
-
return;
|
|
143
|
-
// Main entries table
|
|
144
|
-
this.db.run(`
|
|
145
|
-
CREATE TABLE IF NOT EXISTS memory_entries (
|
|
146
|
-
id TEXT PRIMARY KEY,
|
|
147
|
-
key TEXT NOT NULL,
|
|
148
|
-
content TEXT NOT NULL,
|
|
149
|
-
embedding BLOB,
|
|
150
|
-
type TEXT NOT NULL,
|
|
151
|
-
namespace TEXT NOT NULL,
|
|
152
|
-
tags TEXT NOT NULL,
|
|
153
|
-
metadata TEXT NOT NULL,
|
|
154
|
-
owner_id TEXT,
|
|
155
|
-
access_level TEXT NOT NULL,
|
|
156
|
-
created_at INTEGER NOT NULL,
|
|
157
|
-
updated_at INTEGER NOT NULL,
|
|
158
|
-
expires_at INTEGER,
|
|
159
|
-
event_at INTEGER,
|
|
160
|
-
version INTEGER NOT NULL DEFAULT 1,
|
|
161
|
-
"references" TEXT NOT NULL,
|
|
162
|
-
access_count INTEGER NOT NULL DEFAULT 0,
|
|
163
|
-
last_accessed_at INTEGER NOT NULL
|
|
164
|
-
)
|
|
165
|
-
`);
|
|
166
|
-
// Indexes for performance
|
|
167
|
-
this.db.run('CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace)');
|
|
168
|
-
this.db.run('CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key)');
|
|
169
|
-
this.db.run('CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type)');
|
|
170
|
-
this.db.run('CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at)');
|
|
171
|
-
this.db.run('CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at)');
|
|
172
|
-
this.db.run('CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key)');
|
|
173
|
-
if (this.config.verbose) {
|
|
174
|
-
console.log('[SqlJsBackend] Schema created successfully');
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Store a memory entry
|
|
179
|
-
*/
|
|
180
|
-
async store(entry) {
|
|
181
|
-
this.ensureInitialized();
|
|
182
|
-
const startTime = performance.now();
|
|
183
|
-
const stmt = `
|
|
184
|
-
INSERT OR REPLACE INTO memory_entries (
|
|
185
|
-
id, key, content, embedding, type, namespace, tags, metadata,
|
|
186
|
-
owner_id, access_level, created_at, updated_at, expires_at, event_at,
|
|
187
|
-
version, "references", access_count, last_accessed_at
|
|
188
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
189
|
-
`;
|
|
190
|
-
const embeddingBuffer = entry.embedding
|
|
191
|
-
? Buffer.from(entry.embedding.buffer)
|
|
192
|
-
: null;
|
|
193
|
-
this.db.run(stmt, [
|
|
194
|
-
entry.id,
|
|
195
|
-
entry.key,
|
|
196
|
-
entry.content,
|
|
197
|
-
embeddingBuffer,
|
|
198
|
-
entry.type,
|
|
199
|
-
entry.namespace,
|
|
200
|
-
JSON.stringify(entry.tags),
|
|
201
|
-
JSON.stringify(entry.metadata),
|
|
202
|
-
entry.ownerId || null,
|
|
203
|
-
entry.accessLevel,
|
|
204
|
-
entry.createdAt,
|
|
205
|
-
entry.updatedAt,
|
|
206
|
-
entry.expiresAt || null,
|
|
207
|
-
entry.eventAt ?? null,
|
|
208
|
-
entry.version,
|
|
209
|
-
JSON.stringify(entry.references),
|
|
210
|
-
entry.accessCount,
|
|
211
|
-
entry.lastAccessedAt,
|
|
212
|
-
]);
|
|
213
|
-
const duration = performance.now() - startTime;
|
|
214
|
-
this.stats.writeCount++;
|
|
215
|
-
this.stats.totalWriteTime += duration;
|
|
216
|
-
this.emit('entry:stored', { entry, duration });
|
|
217
|
-
}
|
|
218
|
-
/**
|
|
219
|
-
* Retrieve a memory entry by ID
|
|
220
|
-
*/
|
|
221
|
-
async get(id) {
|
|
222
|
-
this.ensureInitialized();
|
|
223
|
-
const startTime = performance.now();
|
|
224
|
-
const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
|
|
225
|
-
const row = stmt.getAsObject([id]);
|
|
226
|
-
stmt.free();
|
|
227
|
-
const duration = performance.now() - startTime;
|
|
228
|
-
this.stats.queryCount++;
|
|
229
|
-
this.stats.totalQueryTime += duration;
|
|
230
|
-
if (!row || Object.keys(row).length === 0) {
|
|
231
|
-
return null;
|
|
232
|
-
}
|
|
233
|
-
const entry = this.rowToEntry(row);
|
|
234
|
-
// Update access tracking
|
|
235
|
-
this.updateAccessTracking(id);
|
|
236
|
-
this.emit('entry:retrieved', { id, duration });
|
|
237
|
-
return entry;
|
|
238
|
-
}
|
|
239
|
-
/**
|
|
240
|
-
* Retrieve a memory entry by key within a namespace
|
|
241
|
-
*/
|
|
242
|
-
async getByKey(namespace, key) {
|
|
243
|
-
this.ensureInitialized();
|
|
244
|
-
const startTime = performance.now();
|
|
245
|
-
const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE namespace = ? AND key = ?');
|
|
246
|
-
const row = stmt.getAsObject([namespace, key]);
|
|
247
|
-
stmt.free();
|
|
248
|
-
const duration = performance.now() - startTime;
|
|
249
|
-
this.stats.queryCount++;
|
|
250
|
-
this.stats.totalQueryTime += duration;
|
|
251
|
-
if (!row || Object.keys(row).length === 0) {
|
|
252
|
-
return null;
|
|
253
|
-
}
|
|
254
|
-
const entry = this.rowToEntry(row);
|
|
255
|
-
// Update access tracking
|
|
256
|
-
this.updateAccessTracking(entry.id);
|
|
257
|
-
this.emit('entry:retrieved', { namespace, key, duration });
|
|
258
|
-
return entry;
|
|
259
|
-
}
|
|
260
|
-
/**
|
|
261
|
-
* Update a memory entry
|
|
262
|
-
*/
|
|
263
|
-
async update(id, updateData) {
|
|
264
|
-
this.ensureInitialized();
|
|
265
|
-
const startTime = performance.now();
|
|
266
|
-
// Get existing entry
|
|
267
|
-
const existing = await this.get(id);
|
|
268
|
-
if (!existing)
|
|
269
|
-
return null;
|
|
270
|
-
// Merge updates
|
|
271
|
-
const updated = {
|
|
272
|
-
...existing,
|
|
273
|
-
...updateData,
|
|
274
|
-
updatedAt: Date.now(),
|
|
275
|
-
version: existing.version + 1,
|
|
276
|
-
};
|
|
277
|
-
// Store updated entry
|
|
278
|
-
await this.store(updated);
|
|
279
|
-
const duration = performance.now() - startTime;
|
|
280
|
-
this.emit('entry:updated', { id, update: updateData, duration });
|
|
281
|
-
return updated;
|
|
282
|
-
}
|
|
283
|
-
/**
|
|
284
|
-
* Delete a memory entry
|
|
285
|
-
*/
|
|
286
|
-
async delete(id) {
|
|
287
|
-
this.ensureInitialized();
|
|
288
|
-
const startTime = performance.now();
|
|
289
|
-
this.db.run('DELETE FROM memory_entries WHERE id = ?', [id]);
|
|
290
|
-
const duration = performance.now() - startTime;
|
|
291
|
-
this.stats.writeCount++;
|
|
292
|
-
this.stats.totalWriteTime += duration;
|
|
293
|
-
this.emit('entry:deleted', { id, duration });
|
|
294
|
-
return true;
|
|
295
|
-
}
|
|
296
|
-
/**
|
|
297
|
-
* Query memory entries
|
|
298
|
-
*/
|
|
299
|
-
async query(query) {
|
|
300
|
-
this.ensureInitialized();
|
|
301
|
-
const startTime = performance.now();
|
|
302
|
-
let sql = 'SELECT * FROM memory_entries WHERE 1=1';
|
|
303
|
-
const params = [];
|
|
304
|
-
// Namespace filter
|
|
305
|
-
if (query.namespace) {
|
|
306
|
-
sql += ' AND namespace = ?';
|
|
307
|
-
params.push(query.namespace);
|
|
308
|
-
}
|
|
309
|
-
// Type filter
|
|
310
|
-
if (query.memoryType) {
|
|
311
|
-
sql += ' AND type = ?';
|
|
312
|
-
params.push(query.memoryType);
|
|
313
|
-
}
|
|
314
|
-
// Owner filter
|
|
315
|
-
if (query.ownerId) {
|
|
316
|
-
sql += ' AND owner_id = ?';
|
|
317
|
-
params.push(query.ownerId);
|
|
318
|
-
}
|
|
319
|
-
// Access level filter
|
|
320
|
-
if (query.accessLevel) {
|
|
321
|
-
sql += ' AND access_level = ?';
|
|
322
|
-
params.push(query.accessLevel);
|
|
323
|
-
}
|
|
324
|
-
// Key filters
|
|
325
|
-
if (query.key) {
|
|
326
|
-
sql += ' AND key = ?';
|
|
327
|
-
params.push(query.key);
|
|
328
|
-
}
|
|
329
|
-
else if (query.keyPrefix) {
|
|
330
|
-
sql += ' AND key LIKE ?';
|
|
331
|
-
params.push(query.keyPrefix + '%');
|
|
332
|
-
}
|
|
333
|
-
// Time range filters
|
|
334
|
-
if (query.createdAfter) {
|
|
335
|
-
sql += ' AND created_at >= ?';
|
|
336
|
-
params.push(query.createdAfter);
|
|
337
|
-
}
|
|
338
|
-
if (query.createdBefore) {
|
|
339
|
-
sql += ' AND created_at <= ?';
|
|
340
|
-
params.push(query.createdBefore);
|
|
341
|
-
}
|
|
342
|
-
if (query.updatedAfter) {
|
|
343
|
-
sql += ' AND updated_at >= ?';
|
|
344
|
-
params.push(query.updatedAfter);
|
|
345
|
-
}
|
|
346
|
-
if (query.updatedBefore) {
|
|
347
|
-
sql += ' AND updated_at <= ?';
|
|
348
|
-
params.push(query.updatedBefore);
|
|
349
|
-
}
|
|
350
|
-
// Event-time range filters (bi-temporal)
|
|
351
|
-
if (query.eventAfter) {
|
|
352
|
-
sql += ' AND event_at >= ?';
|
|
353
|
-
params.push(query.eventAfter);
|
|
354
|
-
}
|
|
355
|
-
if (query.eventBefore) {
|
|
356
|
-
sql += ' AND event_at <= ?';
|
|
357
|
-
params.push(query.eventBefore);
|
|
358
|
-
}
|
|
359
|
-
// Expiration filter
|
|
360
|
-
if (!query.includeExpired) {
|
|
361
|
-
sql += ' AND (expires_at IS NULL OR expires_at > ?)';
|
|
362
|
-
params.push(Date.now());
|
|
363
|
-
}
|
|
364
|
-
// Ordering and pagination
|
|
365
|
-
const colMap = {
|
|
366
|
-
createdAt: 'created_at', updatedAt: 'updated_at',
|
|
367
|
-
lastAccessedAt: 'last_accessed_at', accessCount: 'access_count', key: 'key',
|
|
368
|
-
};
|
|
369
|
-
const orderCol = (query.sortField && query.sortField !== 'score' && colMap[query.sortField])
|
|
370
|
-
? colMap[query.sortField]
|
|
371
|
-
: 'created_at';
|
|
372
|
-
const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
|
|
373
|
-
sql += ` ORDER BY ${orderCol} ${orderDir}`;
|
|
374
|
-
if (query.limit) {
|
|
375
|
-
sql += ' LIMIT ?';
|
|
376
|
-
params.push(query.limit);
|
|
377
|
-
}
|
|
378
|
-
if (query.offset) {
|
|
379
|
-
sql += ' OFFSET ?';
|
|
380
|
-
params.push(query.offset);
|
|
381
|
-
}
|
|
382
|
-
const stmt = this.db.prepare(sql);
|
|
383
|
-
if (params.length > 0) {
|
|
384
|
-
stmt.bind(params);
|
|
385
|
-
}
|
|
386
|
-
const results = [];
|
|
387
|
-
while (stmt.step()) {
|
|
388
|
-
const row = stmt.getAsObject();
|
|
389
|
-
const entry = this.rowToEntry(row);
|
|
390
|
-
// Tag filtering (post-query since tags are JSON)
|
|
391
|
-
if (query.tags && query.tags.length > 0) {
|
|
392
|
-
const hasAllTags = query.tags.every((tag) => entry.tags.includes(tag));
|
|
393
|
-
if (!hasAllTags)
|
|
394
|
-
continue;
|
|
395
|
-
}
|
|
396
|
-
// Metadata filtering (post-query since metadata is JSON)
|
|
397
|
-
if (query.metadata) {
|
|
398
|
-
const matchesMetadata = Object.entries(query.metadata).every(([key, value]) => entry.metadata[key] === value);
|
|
399
|
-
if (!matchesMetadata)
|
|
400
|
-
continue;
|
|
401
|
-
}
|
|
402
|
-
results.push(entry);
|
|
403
|
-
}
|
|
404
|
-
stmt.free();
|
|
405
|
-
const duration = performance.now() - startTime;
|
|
406
|
-
this.stats.queryCount++;
|
|
407
|
-
this.stats.totalQueryTime += duration;
|
|
408
|
-
this.emit('query:executed', { query, resultCount: results.length, duration });
|
|
409
|
-
return results;
|
|
410
|
-
}
|
|
411
|
-
/**
|
|
412
|
-
* Semantic vector search (limited without vector index)
|
|
413
|
-
*/
|
|
414
|
-
async search(embedding, options) {
|
|
415
|
-
this.ensureInitialized();
|
|
416
|
-
// Get all entries with embeddings — namespace-scoped when the caller
|
|
417
|
-
// filtered (previously ignored, so cross-namespace hits leaked in and the
|
|
418
|
-
// arbitrary 1000-row cap silently dropped later entries).
|
|
419
|
-
const entries = await this.query({
|
|
420
|
-
type: 'hybrid',
|
|
421
|
-
namespace: options.filters?.namespace,
|
|
422
|
-
limit: options.filters?.limit || 50000,
|
|
423
|
-
});
|
|
424
|
-
// Calculate cosine similarity for each entry
|
|
425
|
-
const results = [];
|
|
426
|
-
for (const entry of entries) {
|
|
427
|
-
if (!entry.embedding)
|
|
428
|
-
continue;
|
|
429
|
-
const similarity = cosineSimilarity(embedding, entry.embedding);
|
|
430
|
-
if (options.threshold && similarity < options.threshold) {
|
|
431
|
-
continue;
|
|
432
|
-
}
|
|
433
|
-
results.push({
|
|
434
|
-
entry,
|
|
435
|
-
score: similarity,
|
|
436
|
-
distance: 1 - similarity,
|
|
437
|
-
});
|
|
438
|
-
}
|
|
439
|
-
// Sort by score descending
|
|
440
|
-
results.sort((a, b) => b.score - a.score);
|
|
441
|
-
// Return top k results
|
|
442
|
-
return results.slice(0, options.k);
|
|
443
|
-
}
|
|
444
|
-
/**
|
|
445
|
-
* Bulk insert entries
|
|
446
|
-
*/
|
|
447
|
-
async bulkInsert(entries) {
|
|
448
|
-
this.ensureInitialized();
|
|
449
|
-
for (const entry of entries) {
|
|
450
|
-
await this.store(entry);
|
|
451
|
-
}
|
|
452
|
-
this.emit('bulk:inserted', { count: entries.length });
|
|
453
|
-
}
|
|
454
|
-
/**
|
|
455
|
-
* Bulk delete entries
|
|
456
|
-
*/
|
|
457
|
-
async bulkDelete(ids) {
|
|
458
|
-
this.ensureInitialized();
|
|
459
|
-
let count = 0;
|
|
460
|
-
for (const id of ids) {
|
|
461
|
-
const success = await this.delete(id);
|
|
462
|
-
if (success)
|
|
463
|
-
count++;
|
|
464
|
-
}
|
|
465
|
-
this.emit('bulk:deleted', { count });
|
|
466
|
-
return count;
|
|
467
|
-
}
|
|
468
|
-
/**
|
|
469
|
-
* Get entry count
|
|
470
|
-
*/
|
|
471
|
-
async count(namespace) {
|
|
472
|
-
this.ensureInitialized();
|
|
473
|
-
let sql = 'SELECT COUNT(*) as count FROM memory_entries';
|
|
474
|
-
const params = [];
|
|
475
|
-
if (namespace) {
|
|
476
|
-
sql += ' WHERE namespace = ?';
|
|
477
|
-
params.push(namespace);
|
|
478
|
-
}
|
|
479
|
-
const stmt = this.db.prepare(sql);
|
|
480
|
-
const row = stmt.getAsObject(params);
|
|
481
|
-
stmt.free();
|
|
482
|
-
return row.count || 0;
|
|
483
|
-
}
|
|
484
|
-
/**
|
|
485
|
-
* List all namespaces
|
|
486
|
-
*/
|
|
487
|
-
async listNamespaces() {
|
|
488
|
-
this.ensureInitialized();
|
|
489
|
-
const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
|
|
490
|
-
const namespaces = [];
|
|
491
|
-
while (stmt.step()) {
|
|
492
|
-
const row = stmt.getAsObject();
|
|
493
|
-
namespaces.push(row.namespace);
|
|
494
|
-
}
|
|
495
|
-
stmt.free();
|
|
496
|
-
return namespaces;
|
|
497
|
-
}
|
|
498
|
-
/**
|
|
499
|
-
* Clear all entries in a namespace
|
|
500
|
-
*/
|
|
501
|
-
async clearNamespace(namespace) {
|
|
502
|
-
this.ensureInitialized();
|
|
503
|
-
const countBefore = await this.count(namespace);
|
|
504
|
-
this.db.run('DELETE FROM memory_entries WHERE namespace = ?', [namespace]);
|
|
505
|
-
this.emit('namespace:cleared', { namespace, count: countBefore });
|
|
506
|
-
return countBefore;
|
|
507
|
-
}
|
|
508
|
-
/**
|
|
509
|
-
* Get backend statistics
|
|
510
|
-
*/
|
|
511
|
-
async getStats() {
|
|
512
|
-
this.ensureInitialized();
|
|
513
|
-
const total = await this.count();
|
|
514
|
-
// Count by namespace
|
|
515
|
-
const entriesByNamespace = {};
|
|
516
|
-
const namespaces = await this.listNamespaces();
|
|
517
|
-
for (const ns of namespaces) {
|
|
518
|
-
entriesByNamespace[ns] = await this.count(ns);
|
|
519
|
-
}
|
|
520
|
-
// Count by type
|
|
521
|
-
const entriesByType = {};
|
|
522
|
-
const types = ['episodic', 'semantic', 'working', 'cache'];
|
|
523
|
-
for (const type of types) {
|
|
524
|
-
const stmt = this.db.prepare('SELECT COUNT(*) as count FROM memory_entries WHERE type = ?');
|
|
525
|
-
const row = stmt.getAsObject([type]);
|
|
526
|
-
stmt.free();
|
|
527
|
-
entriesByType[type] = row.count || 0;
|
|
528
|
-
}
|
|
529
|
-
return {
|
|
530
|
-
totalEntries: total,
|
|
531
|
-
entriesByNamespace,
|
|
532
|
-
entriesByType,
|
|
533
|
-
memoryUsage: this.estimateMemoryUsage(),
|
|
534
|
-
avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0,
|
|
535
|
-
avgSearchTime: 0, // Not tracked separately
|
|
536
|
-
};
|
|
537
|
-
}
|
|
538
|
-
/**
|
|
539
|
-
* Perform health check
|
|
540
|
-
*/
|
|
541
|
-
async healthCheck() {
|
|
542
|
-
const issues = [];
|
|
543
|
-
const recommendations = [];
|
|
544
|
-
// Storage health
|
|
545
|
-
const storageStart = performance.now();
|
|
546
|
-
const storageHealthy = this.db !== null;
|
|
547
|
-
const storageLatency = performance.now() - storageStart;
|
|
548
|
-
if (!storageHealthy) {
|
|
549
|
-
issues.push('Database not initialized');
|
|
550
|
-
}
|
|
551
|
-
// Index health (sql.js doesn't have native vector index)
|
|
552
|
-
const indexHealth = {
|
|
553
|
-
status: 'healthy',
|
|
554
|
-
latency: 0,
|
|
555
|
-
message: 'No vector index (brute-force search)',
|
|
556
|
-
};
|
|
557
|
-
recommendations.push('Consider using better-sqlite3 with HNSW for faster vector search');
|
|
558
|
-
// Cache health (not applicable for sql.js)
|
|
559
|
-
const cacheHealth = {
|
|
560
|
-
status: 'healthy',
|
|
561
|
-
latency: 0,
|
|
562
|
-
message: 'No separate cache layer',
|
|
563
|
-
};
|
|
564
|
-
const status = issues.length === 0 ? 'healthy' : 'degraded';
|
|
565
|
-
return {
|
|
566
|
-
status,
|
|
567
|
-
components: {
|
|
568
|
-
storage: {
|
|
569
|
-
status: storageHealthy ? 'healthy' : 'unhealthy',
|
|
570
|
-
latency: storageLatency,
|
|
571
|
-
},
|
|
572
|
-
index: indexHealth,
|
|
573
|
-
cache: cacheHealth,
|
|
574
|
-
},
|
|
575
|
-
timestamp: Date.now(),
|
|
576
|
-
issues,
|
|
577
|
-
recommendations,
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
/**
|
|
581
|
-
* Persist changes to disk (sql.js is in-memory, needs explicit save)
|
|
582
|
-
*/
|
|
583
|
-
async persist() {
|
|
584
|
-
if (!this.db || this.config.databasePath === ':memory:') {
|
|
585
|
-
return;
|
|
586
|
-
}
|
|
587
|
-
const data = this.db.export();
|
|
588
|
-
const buffer = Buffer.from(data);
|
|
589
|
-
writeFileAtomicSync(this.config.databasePath, buffer);
|
|
590
|
-
if (this.config.verbose) {
|
|
591
|
-
console.log(`[SqlJsBackend] Persisted ${buffer.length} bytes to ${this.config.databasePath}`);
|
|
592
|
-
}
|
|
593
|
-
this.emit('persisted', { size: buffer.length, path: this.config.databasePath });
|
|
594
|
-
}
|
|
595
|
-
// ===== Helper Methods =====
|
|
596
|
-
ensureInitialized() {
|
|
597
|
-
if (!this.initialized || !this.db) {
|
|
598
|
-
throw new Error('SqlJsBackend not initialized. Call initialize() first.');
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
rowToEntry(row) {
|
|
602
|
-
return {
|
|
603
|
-
id: row.id,
|
|
604
|
-
key: row.key,
|
|
605
|
-
content: row.content,
|
|
606
|
-
embedding: row.embedding
|
|
607
|
-
? new Float32Array(new Uint8Array(row.embedding).buffer)
|
|
608
|
-
: undefined,
|
|
609
|
-
type: row.type,
|
|
610
|
-
namespace: row.namespace,
|
|
611
|
-
tags: JSON.parse(row.tags),
|
|
612
|
-
metadata: JSON.parse(row.metadata),
|
|
613
|
-
ownerId: row.owner_id,
|
|
614
|
-
accessLevel: row.access_level,
|
|
615
|
-
createdAt: row.created_at,
|
|
616
|
-
updatedAt: row.updated_at,
|
|
617
|
-
expiresAt: row.expires_at,
|
|
618
|
-
eventAt: row.event_at ?? undefined,
|
|
619
|
-
version: row.version,
|
|
620
|
-
references: JSON.parse(row.references),
|
|
621
|
-
accessCount: row.access_count,
|
|
622
|
-
lastAccessedAt: row.last_accessed_at,
|
|
623
|
-
};
|
|
624
|
-
}
|
|
625
|
-
updateAccessTracking(id) {
|
|
626
|
-
if (!this.db)
|
|
627
|
-
return;
|
|
628
|
-
this.db.run('UPDATE memory_entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id = ?', [Date.now(), id]);
|
|
629
|
-
}
|
|
630
|
-
// cosineSimilarity removed — use shared math-utils.ts implementation
|
|
631
|
-
estimateMemoryUsage() {
|
|
632
|
-
if (!this.db)
|
|
633
|
-
return 0;
|
|
634
|
-
// Cheap on-disk-size estimate via PRAGMA — avoids export()-ing the whole
|
|
635
|
-
// database (which would transiently double WASM heap memory) just to
|
|
636
|
-
// measure its size.
|
|
637
|
-
const pageCount = this.pragmaValue('page_count');
|
|
638
|
-
const pageSize = this.pragmaValue('page_size');
|
|
639
|
-
return pageCount * pageSize;
|
|
640
|
-
}
|
|
641
|
-
pragmaValue(pragma) {
|
|
642
|
-
if (!this.db)
|
|
643
|
-
return 0;
|
|
644
|
-
const stmt = this.db.prepare(`PRAGMA ${pragma}`);
|
|
645
|
-
stmt.step();
|
|
646
|
-
const row = stmt.getAsObject();
|
|
647
|
-
stmt.free();
|
|
648
|
-
return row[pragma] || 0;
|
|
112
|
+
// SqlBackend.shutdown() persists before closing.
|
|
113
|
+
await super.shutdown();
|
|
649
114
|
}
|
|
650
115
|
}
|
|
116
|
+
export default SqlJsBackend;
|
|
651
117
|
//# sourceMappingURL=sqljs-backend.js.map
|