moflo 4.9.36 → 4.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-agent-rules.md +12 -0
- package/.claude/guidance/shipped/moflo-memory-protocol.md +34 -0
- package/.claude/guidance/shipped/moflo-memorydb-maintenance.md +22 -11
- package/.claude/guidance/shipped/moflo-root-cause-discipline.md +47 -0
- package/.claude/guidance/shipped/moflo-subagents.md +4 -0
- package/.claude/helpers/gate.cjs +3 -3
- package/.claude/helpers/statusline.cjs +69 -33
- package/.claude/helpers/subagent-bootstrap.json +1 -1
- package/.claude/helpers/subagent-start.cjs +1 -1
- package/.claude/skills/eldar/SKILL.md +8 -0
- package/bin/build-embeddings.mjs +6 -20
- package/bin/cli.js +5 -0
- package/bin/gate.cjs +3 -3
- package/bin/generate-code-map.mjs +4 -24
- package/bin/hooks.mjs +3 -12
- package/bin/index-all.mjs +3 -13
- package/bin/index-guidance.mjs +59 -119
- package/bin/index-patterns.mjs +6 -24
- package/bin/index-tests.mjs +4 -23
- package/bin/lib/db-repair.mjs +4 -25
- package/bin/lib/get-backend.mjs +306 -0
- package/bin/lib/incremental-write.mjs +27 -7
- package/bin/lib/moflo-paths.mjs +64 -4
- package/bin/lib/suppress-sqlite-warning.mjs +57 -0
- package/bin/migrations/knowledge-purge.mjs +7 -8
- package/bin/migrations/knowledge-to-learnings.mjs +7 -9
- package/bin/migrations/purge-doc-entries.mjs +52 -0
- package/bin/migrations/strip-context-preambles.mjs +95 -0
- package/bin/run-migrations.mjs +1 -10
- package/bin/semantic-search.mjs +11 -19
- package/bin/session-start-launcher.mjs +102 -100
- package/bin/simplify-classify.cjs +38 -17
- package/dist/src/cli/commands/daemon.js +38 -11
- package/dist/src/cli/commands/doctor-checks-coverage-truth.js +136 -0
- package/dist/src/cli/commands/doctor-checks-memory-access.js +244 -5
- package/dist/src/cli/commands/doctor-checks-memory.js +13 -18
- package/dist/src/cli/commands/doctor-checks-version-skew.js +94 -0
- package/dist/src/cli/commands/doctor-checks-writers-audit.js +170 -0
- package/dist/src/cli/commands/doctor-embedding-hygiene.js +3 -15
- package/dist/src/cli/commands/doctor-fixes.js +30 -0
- package/dist/src/cli/commands/doctor-registry.js +14 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/embeddings.js +17 -22
- package/dist/src/cli/commands/memory.js +54 -75
- package/dist/src/cli/embeddings/persistent-cache.js +44 -83
- package/dist/src/cli/init/claudemd-generator.js +4 -0
- package/dist/src/cli/init/moflo-init.js +40 -0
- package/dist/src/cli/mcp-tools/memory-tools.js +177 -32
- package/dist/src/cli/memory/bridge-core.js +256 -30
- package/dist/src/cli/memory/bridge-entries.js +76 -8
- package/dist/src/cli/memory/controller-registry.js +7 -2
- package/dist/src/cli/memory/controllers/batch-operations.js +5 -1
- package/dist/src/cli/memory/controllers/hierarchical-memory.js +7 -2
- package/dist/src/cli/memory/controllers/mutation-guard.js +22 -2
- package/dist/src/cli/memory/daemon-backend.js +400 -0
- package/dist/src/cli/memory/daemon-write-client.js +192 -15
- package/dist/src/cli/memory/database-provider.js +57 -40
- package/dist/src/cli/memory/hnsw-persistence.js +6 -8
- package/dist/src/cli/memory/index.js +0 -1
- package/dist/src/cli/memory/memory-bridge.js +40 -8
- package/dist/src/cli/memory/memory-initializer.js +286 -220
- package/dist/src/cli/memory/rvf-migration.js +25 -11
- package/dist/src/cli/memory/sqlite-backend.js +573 -0
- package/dist/src/cli/memory/suppress-sqlite-warning.js +49 -0
- package/dist/src/cli/services/cherry-pick-learnings.js +32 -21
- package/dist/src/cli/services/daemon-dashboard.js +13 -1
- package/dist/src/cli/services/daemon-lock.js +58 -1
- package/dist/src/cli/services/daemon-memory-rpc.js +245 -10
- package/dist/src/cli/services/embeddings-migration.js +9 -12
- package/dist/src/cli/services/ephemeral-namespace-purge.js +6 -11
- package/dist/src/cli/services/learning-service.js +12 -20
- package/dist/src/cli/services/project-root.js +69 -9
- package/dist/src/cli/services/soft-delete-purge.js +6 -11
- package/dist/src/cli/services/sqljs-migration-store.js +4 -1
- package/dist/src/cli/services/subagent-bootstrap.js +1 -1
- package/dist/src/cli/shared/events/event-store.js +26 -55
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -4
- package/dist/src/cli/memory/sqljs-backend.js +0 -643
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* RVF Migration Utility — bidirectional migration between RVF and legacy
|
|
3
|
-
* formats (JSON files,
|
|
3
|
+
* formats (JSON files, node:sqlite databases).
|
|
4
4
|
* @module moflo/cli/memory/rvf-migration
|
|
5
5
|
*/
|
|
6
6
|
import { readFile, writeFile, rename, unlink, mkdir } from 'node:fs/promises';
|
|
7
7
|
import { existsSync } from 'node:fs';
|
|
8
8
|
import { dirname, resolve } from 'node:path';
|
|
9
9
|
import { RvfBackend } from './rvf-backend.js';
|
|
10
|
+
import { openDaemonDatabase } from './daemon-backend.js';
|
|
10
11
|
import { generateMemoryId } from './types.js';
|
|
11
12
|
// -- Internal helpers -------------------------------------------------------
|
|
12
13
|
function fillDefaults(raw) {
|
|
@@ -106,18 +107,31 @@ function normalizeSqliteRow(row) {
|
|
|
106
107
|
}
|
|
107
108
|
return out;
|
|
108
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Read `memory_entries` rows out of a legacy SQLite file via the unified
|
|
112
|
+
* `openDaemonDatabase` factory. node:sqlite's built-in WAL handling means
|
|
113
|
+
* the read is non-destructive to any concurrent writer.
|
|
114
|
+
*/
|
|
109
115
|
async function readSqliteRows(dbPath) {
|
|
110
|
-
|
|
116
|
+
const db = openDaemonDatabase(dbPath);
|
|
111
117
|
try {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
+
const result = db.exec(`SELECT id, key, namespace, content, type, embedding, embedding_model,
|
|
119
|
+
embedding_dimensions, tags, metadata, owner_id, created_at,
|
|
120
|
+
updated_at, expires_at, last_accessed_at, access_count, status
|
|
121
|
+
FROM memory_entries`);
|
|
122
|
+
if (!result[0]?.values?.length)
|
|
123
|
+
return [];
|
|
124
|
+
const columns = result[0].columns;
|
|
125
|
+
return result[0].values.map((row) => {
|
|
126
|
+
const obj = {};
|
|
127
|
+
for (let i = 0; i < columns.length; i++) {
|
|
128
|
+
obj[columns[i]] = row[i];
|
|
129
|
+
}
|
|
130
|
+
return obj;
|
|
131
|
+
});
|
|
118
132
|
}
|
|
119
|
-
|
|
120
|
-
|
|
133
|
+
finally {
|
|
134
|
+
db.close();
|
|
121
135
|
}
|
|
122
136
|
}
|
|
123
137
|
// -- Batch migration helper -------------------------------------------------
|
|
@@ -175,7 +189,7 @@ export class RvfMigrator {
|
|
|
175
189
|
console.log(`[RvfMigrator] Migrated ${migrated} entries from JSON to RVF`);
|
|
176
190
|
return mkResult(errors.length === 0, migrated, 'json', 'rvf', start, errors);
|
|
177
191
|
}
|
|
178
|
-
/** Migrate a SQLite (
|
|
192
|
+
/** Migrate a SQLite (node:sqlite) database to RVF. */
|
|
179
193
|
static async fromSqlite(dbPath, rvfPath, options = {}) {
|
|
180
194
|
const start = Date.now();
|
|
181
195
|
let rows;
|
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SqliteBackend — native node:sqlite backend for moflo.db
|
|
3
|
+
*
|
|
4
|
+
* Drop-in replacement for {@link SqlJsBackend} (same IMemoryBackend surface,
|
|
5
|
+
* same schema, same event emissions) backed by the Node 22+ built-in
|
|
6
|
+
* `node:sqlite` engine instead of WASM. Phase 4 (#1083) made this the
|
|
7
|
+
* default; Phase 5 (#1084) deletes the sql.js backend + npm dep.
|
|
8
|
+
*
|
|
9
|
+
* Why this exists: epic #1078 / Phase 0 spike (#1079, PR #1085) confirmed
|
|
10
|
+
* `node:sqlite` parity against shipped sql.js DBs. The structural sql.js
|
|
11
|
+
* failure mode is whole-file snapshots: each process holds its own copy and
|
|
12
|
+
* the last flusher wipes the other's writes. `node:sqlite` writes through
|
|
13
|
+
* the OS file handle and uses WAL for multi-process serialization, which
|
|
14
|
+
* removes that failure mode entirely.
|
|
15
|
+
*
|
|
16
|
+
* @module v3/memory/sqlite-backend
|
|
17
|
+
*/
|
|
18
|
+
// MUST come before `import 'node:sqlite'` below — see suppress-sqlite-warning
|
|
19
|
+
// header for rationale (#1098).
|
|
20
|
+
import './suppress-sqlite-warning.js';
|
|
21
|
+
import { EventEmitter } from 'node:events';
|
|
22
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
23
|
+
import { cosineSimilarity } from './hnsw-lite.js';
|
|
24
|
+
const DEFAULT_CONFIG = {
|
|
25
|
+
databasePath: ':memory:',
|
|
26
|
+
optimize: true,
|
|
27
|
+
defaultNamespace: 'default',
|
|
28
|
+
maxEntries: 1000000,
|
|
29
|
+
verbose: false,
|
|
30
|
+
autoPersistInterval: 0,
|
|
31
|
+
};
|
|
32
|
+
export class SqliteBackend extends EventEmitter {
|
|
33
|
+
config;
|
|
34
|
+
db = null;
|
|
35
|
+
initialized = false;
|
|
36
|
+
stmts = {};
|
|
37
|
+
cachedPageSize = 0;
|
|
38
|
+
stats = {
|
|
39
|
+
queryCount: 0,
|
|
40
|
+
totalQueryTime: 0,
|
|
41
|
+
writeCount: 0,
|
|
42
|
+
totalWriteTime: 0,
|
|
43
|
+
};
|
|
44
|
+
constructor(config = {}) {
|
|
45
|
+
super();
|
|
46
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
47
|
+
}
|
|
48
|
+
async initialize() {
|
|
49
|
+
if (this.initialized)
|
|
50
|
+
return;
|
|
51
|
+
const db = new DatabaseSync(this.config.databasePath);
|
|
52
|
+
try {
|
|
53
|
+
if (this.config.databasePath !== ':memory:') {
|
|
54
|
+
// WAL is required for the multi-process serialization invariant proven
|
|
55
|
+
// in the Phase 0 spike. The spike verified the .db-wal/.db-shm sidecars
|
|
56
|
+
// appear on first write.
|
|
57
|
+
//
|
|
58
|
+
// busy_timeout BEFORE journal_mode = WAL: the WAL pragma briefly takes
|
|
59
|
+
// an EXCLUSIVE lock, and concurrent openers otherwise hit "database is
|
|
60
|
+
// locked" with no retry budget (#1097).
|
|
61
|
+
// 15000ms matches daemon-backend.ts (#1098 — the harness's first-pass
|
|
62
|
+
// indexer can hold a write lock for 5–8s after npm install).
|
|
63
|
+
db.exec('PRAGMA busy_timeout = 15000');
|
|
64
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
65
|
+
db.exec('PRAGMA synchronous = NORMAL');
|
|
66
|
+
}
|
|
67
|
+
this.db = db;
|
|
68
|
+
this.createSchema();
|
|
69
|
+
this.prepareCachedStatements();
|
|
70
|
+
this.cachedPageSize = this.readPageSize();
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
// Don't leak the handle if any setup step threw — a subsequent
|
|
74
|
+
// initialize() retry would otherwise orphan it.
|
|
75
|
+
try {
|
|
76
|
+
db.close();
|
|
77
|
+
}
|
|
78
|
+
catch { /* already closed */ }
|
|
79
|
+
this.db = null;
|
|
80
|
+
this.stmts = {};
|
|
81
|
+
throw err;
|
|
82
|
+
}
|
|
83
|
+
this.initialized = true;
|
|
84
|
+
this.emit('initialized');
|
|
85
|
+
if (this.config.verbose) {
|
|
86
|
+
console.log(`[SqliteBackend] Ready (${this.config.databasePath})`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
readPageSize() {
|
|
90
|
+
if (!this.db)
|
|
91
|
+
return 0;
|
|
92
|
+
try {
|
|
93
|
+
const row = this.db.prepare('PRAGMA page_size').get();
|
|
94
|
+
return Number(row?.page_size ?? 0);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async shutdown() {
|
|
101
|
+
if (!this.initialized || !this.db)
|
|
102
|
+
return;
|
|
103
|
+
// Finalize cached statements before closing — node:sqlite requires every
|
|
104
|
+
// prepared statement to be released before db.close().
|
|
105
|
+
this.stmts = {};
|
|
106
|
+
this.db.close();
|
|
107
|
+
this.db = null;
|
|
108
|
+
this.initialized = false;
|
|
109
|
+
this.emit('shutdown');
|
|
110
|
+
}
|
|
111
|
+
createSchema() {
|
|
112
|
+
if (!this.db)
|
|
113
|
+
return;
|
|
114
|
+
// Mirrors SqlJsBackend.createSchema exactly. `IF NOT EXISTS` is a no-op
|
|
115
|
+
// when a pre-existing DB (e.g. one created by MEMORY_SCHEMA_V3) already
|
|
116
|
+
// has the table — the schema discrepancy noted in epic #1078 Phase 0 is
|
|
117
|
+
// tolerated here for drop-in parity.
|
|
118
|
+
this.db.exec(`
|
|
119
|
+
CREATE TABLE IF NOT EXISTS memory_entries (
|
|
120
|
+
id TEXT PRIMARY KEY,
|
|
121
|
+
key TEXT NOT NULL,
|
|
122
|
+
content TEXT NOT NULL,
|
|
123
|
+
embedding BLOB,
|
|
124
|
+
type TEXT NOT NULL,
|
|
125
|
+
namespace TEXT NOT NULL,
|
|
126
|
+
tags TEXT NOT NULL,
|
|
127
|
+
metadata TEXT NOT NULL,
|
|
128
|
+
owner_id TEXT,
|
|
129
|
+
access_level TEXT NOT NULL,
|
|
130
|
+
created_at INTEGER NOT NULL,
|
|
131
|
+
updated_at INTEGER NOT NULL,
|
|
132
|
+
expires_at INTEGER,
|
|
133
|
+
version INTEGER NOT NULL DEFAULT 1,
|
|
134
|
+
"references" TEXT NOT NULL,
|
|
135
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
136
|
+
last_accessed_at INTEGER NOT NULL
|
|
137
|
+
);
|
|
138
|
+
CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace);
|
|
139
|
+
CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key);
|
|
140
|
+
CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type);
|
|
141
|
+
CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at);
|
|
142
|
+
CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at);
|
|
143
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key);
|
|
144
|
+
`);
|
|
145
|
+
}
|
|
146
|
+
prepareCachedStatements() {
|
|
147
|
+
if (!this.db)
|
|
148
|
+
return;
|
|
149
|
+
this.stmts.store = this.db.prepare(`
|
|
150
|
+
INSERT OR REPLACE INTO memory_entries (
|
|
151
|
+
id, key, content, embedding, type, namespace, tags, metadata,
|
|
152
|
+
owner_id, access_level, created_at, updated_at, expires_at,
|
|
153
|
+
version, "references", access_count, last_accessed_at
|
|
154
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
155
|
+
`);
|
|
156
|
+
this.stmts.getById = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
|
|
157
|
+
this.stmts.getByKey = this.db.prepare('SELECT * FROM memory_entries WHERE namespace = ? AND key = ?');
|
|
158
|
+
this.stmts.deleteById = this.db.prepare('DELETE FROM memory_entries WHERE id = ?');
|
|
159
|
+
this.stmts.updateAccess = this.db.prepare('UPDATE memory_entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id = ?');
|
|
160
|
+
this.stmts.countAll = this.db.prepare('SELECT COUNT(*) AS c FROM memory_entries');
|
|
161
|
+
this.stmts.countByNamespace = this.db.prepare('SELECT COUNT(*) AS c FROM memory_entries WHERE namespace = ?');
|
|
162
|
+
this.stmts.countByType = this.db.prepare('SELECT COUNT(*) AS c FROM memory_entries WHERE type = ?');
|
|
163
|
+
this.stmts.listNamespaces = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
|
|
164
|
+
this.stmts.clearNamespace = this.db.prepare('SELECT COUNT(*) AS c FROM memory_entries WHERE namespace = ?');
|
|
165
|
+
this.stmts.deleteByNamespace = this.db.prepare('DELETE FROM memory_entries WHERE namespace = ?');
|
|
166
|
+
}
|
|
167
|
+
async store(entry) {
|
|
168
|
+
this.ensureInitialized();
|
|
169
|
+
const t0 = performance.now();
|
|
170
|
+
// Copy into a fresh Buffer rather than reusing entry.embedding.buffer —
|
|
171
|
+
// the underlying ArrayBufferLike may be a SharedArrayBuffer (typing
|
|
172
|
+
// mismatch with node:sqlite's Buffer expectation) and downstream code
|
|
173
|
+
// shouldn't be coupled to whatever allocation backed the input.
|
|
174
|
+
let embeddingBuf = null;
|
|
175
|
+
if (entry.embedding) {
|
|
176
|
+
embeddingBuf = Buffer.alloc(entry.embedding.byteLength);
|
|
177
|
+
embeddingBuf.set(new Uint8Array(entry.embedding.buffer.slice(entry.embedding.byteOffset, entry.embedding.byteOffset + entry.embedding.byteLength)));
|
|
178
|
+
}
|
|
179
|
+
this.stmts.store.run(entry.id, entry.key, entry.content, embeddingBuf, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId ?? null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt ?? null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt);
|
|
180
|
+
const duration = performance.now() - t0;
|
|
181
|
+
this.stats.writeCount++;
|
|
182
|
+
this.stats.totalWriteTime += duration;
|
|
183
|
+
this.emit('entry:stored', { entry, duration });
|
|
184
|
+
}
|
|
185
|
+
async get(id) {
|
|
186
|
+
this.ensureInitialized();
|
|
187
|
+
const t0 = performance.now();
|
|
188
|
+
const row = this.stmts.getById.get(id);
|
|
189
|
+
const duration = performance.now() - t0;
|
|
190
|
+
this.stats.queryCount++;
|
|
191
|
+
this.stats.totalQueryTime += duration;
|
|
192
|
+
if (!row)
|
|
193
|
+
return null;
|
|
194
|
+
const entry = this.rowToEntry(row);
|
|
195
|
+
this.updateAccessTracking(id);
|
|
196
|
+
this.emit('entry:retrieved', { id, duration });
|
|
197
|
+
return entry;
|
|
198
|
+
}
|
|
199
|
+
async getByKey(namespace, key) {
|
|
200
|
+
this.ensureInitialized();
|
|
201
|
+
const t0 = performance.now();
|
|
202
|
+
const row = this.stmts.getByKey.get(namespace, key);
|
|
203
|
+
const duration = performance.now() - t0;
|
|
204
|
+
this.stats.queryCount++;
|
|
205
|
+
this.stats.totalQueryTime += duration;
|
|
206
|
+
if (!row)
|
|
207
|
+
return null;
|
|
208
|
+
const entry = this.rowToEntry(row);
|
|
209
|
+
this.updateAccessTracking(entry.id);
|
|
210
|
+
this.emit('entry:retrieved', { namespace, key, duration });
|
|
211
|
+
return entry;
|
|
212
|
+
}
|
|
213
|
+
async update(id, updateData) {
|
|
214
|
+
this.ensureInitialized();
|
|
215
|
+
const t0 = performance.now();
|
|
216
|
+
const existing = await this.get(id);
|
|
217
|
+
if (!existing)
|
|
218
|
+
return null;
|
|
219
|
+
const updated = {
|
|
220
|
+
...existing,
|
|
221
|
+
...updateData,
|
|
222
|
+
updatedAt: Date.now(),
|
|
223
|
+
version: existing.version + 1,
|
|
224
|
+
};
|
|
225
|
+
await this.store(updated);
|
|
226
|
+
const duration = performance.now() - t0;
|
|
227
|
+
this.emit('entry:updated', { id, update: updateData, duration });
|
|
228
|
+
return updated;
|
|
229
|
+
}
|
|
230
|
+
async delete(id) {
|
|
231
|
+
this.ensureInitialized();
|
|
232
|
+
const t0 = performance.now();
|
|
233
|
+
this.stmts.deleteById.run(id);
|
|
234
|
+
const duration = performance.now() - t0;
|
|
235
|
+
this.stats.writeCount++;
|
|
236
|
+
this.stats.totalWriteTime += duration;
|
|
237
|
+
this.emit('entry:deleted', { id, duration });
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
async query(query) {
|
|
241
|
+
this.ensureInitialized();
|
|
242
|
+
const t0 = performance.now();
|
|
243
|
+
let sql = 'SELECT * FROM memory_entries WHERE 1=1';
|
|
244
|
+
const params = [];
|
|
245
|
+
if (query.namespace) {
|
|
246
|
+
sql += ' AND namespace = ?';
|
|
247
|
+
params.push(query.namespace);
|
|
248
|
+
}
|
|
249
|
+
if (query.memoryType) {
|
|
250
|
+
sql += ' AND type = ?';
|
|
251
|
+
params.push(query.memoryType);
|
|
252
|
+
}
|
|
253
|
+
if (query.ownerId) {
|
|
254
|
+
sql += ' AND owner_id = ?';
|
|
255
|
+
params.push(query.ownerId);
|
|
256
|
+
}
|
|
257
|
+
if (query.accessLevel) {
|
|
258
|
+
sql += ' AND access_level = ?';
|
|
259
|
+
params.push(query.accessLevel);
|
|
260
|
+
}
|
|
261
|
+
if (query.key) {
|
|
262
|
+
sql += ' AND key = ?';
|
|
263
|
+
params.push(query.key);
|
|
264
|
+
}
|
|
265
|
+
else if (query.keyPrefix) {
|
|
266
|
+
sql += ' AND key LIKE ?';
|
|
267
|
+
params.push(query.keyPrefix + '%');
|
|
268
|
+
}
|
|
269
|
+
if (query.createdAfter !== undefined) {
|
|
270
|
+
sql += ' AND created_at >= ?';
|
|
271
|
+
params.push(query.createdAfter);
|
|
272
|
+
}
|
|
273
|
+
if (query.createdBefore !== undefined) {
|
|
274
|
+
sql += ' AND created_at <= ?';
|
|
275
|
+
params.push(query.createdBefore);
|
|
276
|
+
}
|
|
277
|
+
if (query.updatedAfter !== undefined) {
|
|
278
|
+
sql += ' AND updated_at >= ?';
|
|
279
|
+
params.push(query.updatedAfter);
|
|
280
|
+
}
|
|
281
|
+
if (query.updatedBefore !== undefined) {
|
|
282
|
+
sql += ' AND updated_at <= ?';
|
|
283
|
+
params.push(query.updatedBefore);
|
|
284
|
+
}
|
|
285
|
+
if (!query.includeExpired) {
|
|
286
|
+
sql += ' AND (expires_at IS NULL OR expires_at > ?)';
|
|
287
|
+
params.push(Date.now());
|
|
288
|
+
}
|
|
289
|
+
sql += ' ORDER BY created_at DESC';
|
|
290
|
+
if (query.limit) {
|
|
291
|
+
sql += ' LIMIT ?';
|
|
292
|
+
params.push(query.limit);
|
|
293
|
+
}
|
|
294
|
+
if (query.offset) {
|
|
295
|
+
sql += ' OFFSET ?';
|
|
296
|
+
params.push(query.offset);
|
|
297
|
+
}
|
|
298
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
299
|
+
const results = [];
|
|
300
|
+
for (const row of rows) {
|
|
301
|
+
const entry = this.rowToEntry(row);
|
|
302
|
+
if (query.tags && query.tags.length > 0) {
|
|
303
|
+
if (!query.tags.every((tag) => entry.tags.includes(tag)))
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
if (query.metadata) {
|
|
307
|
+
const ok = Object.entries(query.metadata).every(([k, v]) => entry.metadata[k] === v);
|
|
308
|
+
if (!ok)
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
results.push(entry);
|
|
312
|
+
}
|
|
313
|
+
const duration = performance.now() - t0;
|
|
314
|
+
this.stats.queryCount++;
|
|
315
|
+
this.stats.totalQueryTime += duration;
|
|
316
|
+
this.emit('query:executed', { query, resultCount: results.length, duration });
|
|
317
|
+
return results;
|
|
318
|
+
}
|
|
319
|
+
async search(embedding, options) {
|
|
320
|
+
this.ensureInitialized();
|
|
321
|
+
const entries = await this.query({
|
|
322
|
+
type: 'hybrid',
|
|
323
|
+
limit: options.filters?.limit ?? 1000,
|
|
324
|
+
});
|
|
325
|
+
const results = [];
|
|
326
|
+
for (const entry of entries) {
|
|
327
|
+
if (!entry.embedding)
|
|
328
|
+
continue;
|
|
329
|
+
const similarity = cosineSimilarity(embedding, entry.embedding);
|
|
330
|
+
if (options.threshold !== undefined && similarity < options.threshold)
|
|
331
|
+
continue;
|
|
332
|
+
results.push({ entry, score: similarity, distance: 1 - similarity });
|
|
333
|
+
}
|
|
334
|
+
results.sort((a, b) => b.score - a.score);
|
|
335
|
+
return results.slice(0, options.k);
|
|
336
|
+
}
|
|
337
|
+
async bulkInsert(entries) {
|
|
338
|
+
this.ensureInitialized();
|
|
339
|
+
// Wrap in a transaction so the whole batch lands atomically and shares
|
|
340
|
+
// one fsync — meaningful win for 100+ entry inserts. Guarded against
|
|
341
|
+
// re-entry: node:sqlite throws on nested BEGIN, so honor an outer
|
|
342
|
+
// transaction if a caller already opened one.
|
|
343
|
+
await this.runInTransaction(async () => {
|
|
344
|
+
for (const entry of entries)
|
|
345
|
+
await this.store(entry);
|
|
346
|
+
});
|
|
347
|
+
this.emit('bulk:inserted', { count: entries.length });
|
|
348
|
+
}
|
|
349
|
+
async bulkDelete(ids) {
|
|
350
|
+
this.ensureInitialized();
|
|
351
|
+
let count = 0;
|
|
352
|
+
await this.runInTransaction(async () => {
|
|
353
|
+
for (const id of ids) {
|
|
354
|
+
const ok = await this.delete(id);
|
|
355
|
+
if (ok)
|
|
356
|
+
count++;
|
|
357
|
+
}
|
|
358
|
+
});
|
|
359
|
+
this.emit('bulk:deleted', { count });
|
|
360
|
+
return count;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Run `fn` inside a transaction, skipping BEGIN/COMMIT when one is already
|
|
364
|
+
* open. node:sqlite throws on nested BEGIN; better-sqlite3 has
|
|
365
|
+
* `db.transaction(fn)` for this, but the built-in engine doesn't.
|
|
366
|
+
*/
|
|
367
|
+
async runInTransaction(fn) {
|
|
368
|
+
const owns = !this.db.isTransaction;
|
|
369
|
+
// BEGIN IMMEDIATE so busy_handler engages on multi-process contention
|
|
370
|
+
// (#1099 — plain BEGIN's read→write upgrade fails fast under WAL).
|
|
371
|
+
if (owns)
|
|
372
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
373
|
+
try {
|
|
374
|
+
await fn();
|
|
375
|
+
if (owns)
|
|
376
|
+
this.db.exec('COMMIT');
|
|
377
|
+
}
|
|
378
|
+
catch (err) {
|
|
379
|
+
if (owns) {
|
|
380
|
+
try {
|
|
381
|
+
this.db.exec('ROLLBACK');
|
|
382
|
+
}
|
|
383
|
+
catch { /* already aborted */ }
|
|
384
|
+
}
|
|
385
|
+
throw err;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
async count(namespace) {
|
|
389
|
+
this.ensureInitialized();
|
|
390
|
+
const row = namespace
|
|
391
|
+
? this.stmts.countByNamespace.get(namespace)
|
|
392
|
+
: this.stmts.countAll.get();
|
|
393
|
+
return Number(row?.c ?? 0);
|
|
394
|
+
}
|
|
395
|
+
async listNamespaces() {
|
|
396
|
+
this.ensureInitialized();
|
|
397
|
+
const rows = this.stmts.listNamespaces.all();
|
|
398
|
+
return rows.map((r) => r.namespace);
|
|
399
|
+
}
|
|
400
|
+
async clearNamespace(namespace) {
|
|
401
|
+
this.ensureInitialized();
|
|
402
|
+
const before = await this.count(namespace);
|
|
403
|
+
this.stmts.deleteByNamespace.run(namespace);
|
|
404
|
+
this.emit('namespace:cleared', { namespace, count: before });
|
|
405
|
+
return before;
|
|
406
|
+
}
|
|
407
|
+
async getStats() {
|
|
408
|
+
this.ensureInitialized();
|
|
409
|
+
// Single GROUP BY scan replaces N namespace queries + 5 type queries
|
|
410
|
+
// (reviewer flag — getStats is called from health checks).
|
|
411
|
+
const rows = this.db
|
|
412
|
+
.prepare('SELECT namespace, type, COUNT(*) AS c FROM memory_entries GROUP BY namespace, type')
|
|
413
|
+
.all();
|
|
414
|
+
const entriesByNamespace = {};
|
|
415
|
+
const entriesByType = {
|
|
416
|
+
episodic: 0, semantic: 0, procedural: 0, working: 0, cache: 0,
|
|
417
|
+
};
|
|
418
|
+
let total = 0;
|
|
419
|
+
for (const row of rows) {
|
|
420
|
+
const ns = String(row.namespace);
|
|
421
|
+
const type = row.type;
|
|
422
|
+
const c = Number(row.c);
|
|
423
|
+
total += c;
|
|
424
|
+
entriesByNamespace[ns] = (entriesByNamespace[ns] ?? 0) + c;
|
|
425
|
+
if (type in entriesByType)
|
|
426
|
+
entriesByType[type] = (entriesByType[type] ?? 0) + c;
|
|
427
|
+
}
|
|
428
|
+
return {
|
|
429
|
+
totalEntries: total,
|
|
430
|
+
entriesByNamespace,
|
|
431
|
+
entriesByType,
|
|
432
|
+
memoryUsage: this.estimateMemoryUsage(),
|
|
433
|
+
avgQueryTime: this.stats.queryCount > 0
|
|
434
|
+
? this.stats.totalQueryTime / this.stats.queryCount
|
|
435
|
+
: 0,
|
|
436
|
+
avgSearchTime: 0,
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
async healthCheck() {
|
|
440
|
+
const issues = [];
|
|
441
|
+
const storageStart = performance.now();
|
|
442
|
+
const storageHealthy = this.db !== null;
|
|
443
|
+
const storageLatency = performance.now() - storageStart;
|
|
444
|
+
if (!storageHealthy)
|
|
445
|
+
issues.push('Database not initialized');
|
|
446
|
+
const indexHealth = {
|
|
447
|
+
status: 'healthy',
|
|
448
|
+
latency: 0,
|
|
449
|
+
message: 'No vector index (brute-force search)',
|
|
450
|
+
};
|
|
451
|
+
const cacheHealth = {
|
|
452
|
+
status: 'healthy',
|
|
453
|
+
latency: 0,
|
|
454
|
+
message: 'No separate cache layer',
|
|
455
|
+
};
|
|
456
|
+
return {
|
|
457
|
+
status: issues.length === 0 ? 'healthy' : 'degraded',
|
|
458
|
+
components: {
|
|
459
|
+
storage: {
|
|
460
|
+
status: storageHealthy ? 'healthy' : 'unhealthy',
|
|
461
|
+
latency: storageLatency,
|
|
462
|
+
},
|
|
463
|
+
index: indexHealth,
|
|
464
|
+
cache: cacheHealth,
|
|
465
|
+
},
|
|
466
|
+
timestamp: Date.now(),
|
|
467
|
+
issues,
|
|
468
|
+
recommendations: ['Consider using MofloDbAdapter for HNSW-indexed vector search'],
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* No-op for parity with {@link SqlJsBackend.persist}. node:sqlite writes
|
|
473
|
+
* straight to the OS file handle — there is no in-memory image to flush.
|
|
474
|
+
*/
|
|
475
|
+
async persist() {
|
|
476
|
+
if (!this.db || this.config.databasePath === ':memory:')
|
|
477
|
+
return;
|
|
478
|
+
// Checkpoint the WAL into the main DB file. The :memory: and readonly
|
|
479
|
+
// cases are already returned above, so a failure here is a real disk-
|
|
480
|
+
// level problem (disk full, locked file, corrupted WAL) — surface it.
|
|
481
|
+
try {
|
|
482
|
+
this.db.exec('PRAGMA wal_checkpoint(PASSIVE)');
|
|
483
|
+
}
|
|
484
|
+
catch (err) {
|
|
485
|
+
if (this.config.verbose) {
|
|
486
|
+
console.warn(`[SqliteBackend] wal_checkpoint failed: ${err.message}`);
|
|
487
|
+
}
|
|
488
|
+
this.emit('error', { operation: 'persist', error: err });
|
|
489
|
+
}
|
|
490
|
+
this.emit('persisted', { path: this.config.databasePath });
|
|
491
|
+
}
|
|
492
|
+
ensureInitialized() {
|
|
493
|
+
if (!this.initialized || !this.db) {
|
|
494
|
+
throw new Error('SqliteBackend not initialized. Call initialize() first.');
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
rowToEntry(row) {
|
|
498
|
+
return {
|
|
499
|
+
id: row.id,
|
|
500
|
+
key: row.key,
|
|
501
|
+
content: row.content,
|
|
502
|
+
embedding: row.embedding ? blobToFloat32(row.embedding, this.config.verbose) : undefined,
|
|
503
|
+
type: row.type,
|
|
504
|
+
namespace: row.namespace,
|
|
505
|
+
tags: JSON.parse(row.tags ?? '[]'),
|
|
506
|
+
metadata: JSON.parse(row.metadata ?? '{}'),
|
|
507
|
+
ownerId: row.owner_id ?? undefined,
|
|
508
|
+
accessLevel: row.access_level,
|
|
509
|
+
createdAt: Number(row.created_at),
|
|
510
|
+
updatedAt: Number(row.updated_at),
|
|
511
|
+
expiresAt: row.expires_at != null ? Number(row.expires_at) : undefined,
|
|
512
|
+
version: Number(row.version ?? 1),
|
|
513
|
+
references: JSON.parse(row.references ?? '[]'),
|
|
514
|
+
accessCount: Number(row.access_count ?? 0),
|
|
515
|
+
lastAccessedAt: Number(row.last_accessed_at ?? 0),
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
updateAccessTracking(id) {
|
|
519
|
+
if (!this.db)
|
|
520
|
+
return;
|
|
521
|
+
this.stmts.updateAccess.run(Date.now(), id);
|
|
522
|
+
}
|
|
523
|
+
estimateMemoryUsage() {
|
|
524
|
+
if (!this.db || !this.cachedPageSize)
|
|
525
|
+
return 0;
|
|
526
|
+
try {
|
|
527
|
+
const row = this.db.prepare('PRAGMA page_count').get();
|
|
528
|
+
return Number(row?.page_count ?? 0) * this.cachedPageSize;
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
return 0;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Convert a stored embedding cell to Float32Array. Handles both BLOB shape
|
|
537
|
+
* (from {@link SqlJsBackend} / this backend) and TEXT-JSON shape (from DBs
|
|
538
|
+
* created via MEMORY_SCHEMA_V3 — the discrepancy called out in epic #1078
|
|
539
|
+
* Phase 0). Logs on malformed inputs because a silently-truncated embedding
|
|
540
|
+
* returns wrong search results.
|
|
541
|
+
*/
|
|
542
|
+
function blobToFloat32(cell, verbose) {
|
|
543
|
+
const toView = (view) => {
|
|
544
|
+
if (view.byteLength % 4 !== 0) {
|
|
545
|
+
if (verbose) {
|
|
546
|
+
console.warn(`[SqliteBackend] embedding BLOB byteLength ${view.byteLength} not aligned to Float32 — returning empty`);
|
|
547
|
+
}
|
|
548
|
+
return new Float32Array(0);
|
|
549
|
+
}
|
|
550
|
+
// Copy into a fresh ArrayBuffer so the returned view is detached from
|
|
551
|
+
// any SharedArrayBuffer the SQLite binding may hand us.
|
|
552
|
+
const copy = new Uint8Array(view.byteLength);
|
|
553
|
+
copy.set(view);
|
|
554
|
+
return new Float32Array(copy.buffer);
|
|
555
|
+
};
|
|
556
|
+
if (cell instanceof Uint8Array)
|
|
557
|
+
return toView(cell);
|
|
558
|
+
if (Buffer.isBuffer(cell))
|
|
559
|
+
return toView(cell);
|
|
560
|
+
if (typeof cell === 'string') {
|
|
561
|
+
try {
|
|
562
|
+
const arr = JSON.parse(cell);
|
|
563
|
+
return new Float32Array(Array.isArray(arr) ? arr : []);
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
if (verbose)
|
|
567
|
+
console.warn('[SqliteBackend] embedding TEXT cell unparseable — returning empty');
|
|
568
|
+
return new Float32Array(0);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
return new Float32Array(0);
|
|
572
|
+
}
|
|
573
|
+
//# sourceMappingURL=sqlite-backend.js.map
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Side-effect module — installs a `process.emitWarning` filter that drops
|
|
3
|
+
* Node's `ExperimentalWarning: SQLite is an experimental feature` line
|
|
4
|
+
* before `node:sqlite` ever fires it.
|
|
5
|
+
*
|
|
6
|
+
* TS twin of `bin/lib/suppress-sqlite-warning.mjs`. Imported at the top of
|
|
7
|
+
* `daemon-backend.ts` so any TS entry point (CLI command, MCP server, vitest
|
|
8
|
+
* test) that pulls in the bridge transitively also gets the filter.
|
|
9
|
+
*
|
|
10
|
+
* See the JS twin for the full rationale; the two MUST stay in lockstep so
|
|
11
|
+
* the symbol-based idempotency guard works whether the first import comes
|
|
12
|
+
* from JS or TS.
|
|
13
|
+
*
|
|
14
|
+
* @module v3/memory/suppress-sqlite-warning
|
|
15
|
+
*/
|
|
16
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
17
|
+
const INSTALLED = Symbol.for('moflo.suppressSqliteWarning.installed');
|
|
18
|
+
function shouldSuppress(message) {
|
|
19
|
+
if (typeof message === 'string') {
|
|
20
|
+
return message.includes('SQLite is an experimental feature');
|
|
21
|
+
}
|
|
22
|
+
if (message && typeof message === 'object') {
|
|
23
|
+
const m = message.message;
|
|
24
|
+
return typeof m === 'string' && m.includes('SQLite is an experimental feature');
|
|
25
|
+
}
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const g = globalThis;
|
|
29
|
+
if (!g[INSTALLED]) {
|
|
30
|
+
g[INSTALLED] = true;
|
|
31
|
+
const originalEmitWarning = process.emitWarning.bind(process);
|
|
32
|
+
// Cast through `any` so we can pass both emit signatures —
|
|
33
|
+
// (message, type, code, ctor) and (message, options) — through unchanged.
|
|
34
|
+
process.emitWarning = function (warning, ...args) {
|
|
35
|
+
if (shouldSuppress(warning))
|
|
36
|
+
return;
|
|
37
|
+
const arg0 = args[0];
|
|
38
|
+
const typeArg = typeof arg0 === 'string'
|
|
39
|
+
? arg0
|
|
40
|
+
: arg0 && typeof arg0 === 'object'
|
|
41
|
+
? arg0.type
|
|
42
|
+
: undefined;
|
|
43
|
+
if (typeArg === 'ExperimentalWarning' && typeof warning === 'string' && warning.includes('SQLite'))
|
|
44
|
+
return;
|
|
45
|
+
originalEmitWarning(warning, ...args);
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export {};
|
|
49
|
+
//# sourceMappingURL=suppress-sqlite-warning.js.map
|