moflo 4.9.36 → 4.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-agent-rules.md +12 -0
- package/.claude/guidance/shipped/moflo-memory-protocol.md +34 -0
- package/.claude/guidance/shipped/moflo-memorydb-maintenance.md +22 -11
- package/.claude/guidance/shipped/moflo-root-cause-discipline.md +47 -0
- package/.claude/guidance/shipped/moflo-subagents.md +4 -0
- package/.claude/helpers/gate.cjs +3 -3
- package/.claude/helpers/statusline.cjs +69 -33
- package/.claude/helpers/subagent-bootstrap.json +1 -1
- package/.claude/helpers/subagent-start.cjs +1 -1
- package/.claude/skills/eldar/SKILL.md +8 -0
- package/bin/build-embeddings.mjs +6 -20
- package/bin/cli.js +5 -0
- package/bin/gate.cjs +3 -3
- package/bin/generate-code-map.mjs +4 -24
- package/bin/hooks.mjs +3 -12
- package/bin/index-all.mjs +3 -13
- package/bin/index-guidance.mjs +59 -119
- package/bin/index-patterns.mjs +6 -24
- package/bin/index-tests.mjs +4 -23
- package/bin/lib/db-repair.mjs +4 -25
- package/bin/lib/get-backend.mjs +306 -0
- package/bin/lib/incremental-write.mjs +27 -7
- package/bin/lib/moflo-paths.mjs +64 -4
- package/bin/lib/suppress-sqlite-warning.mjs +57 -0
- package/bin/migrations/knowledge-purge.mjs +7 -8
- package/bin/migrations/knowledge-to-learnings.mjs +7 -9
- package/bin/migrations/purge-doc-entries.mjs +52 -0
- package/bin/migrations/strip-context-preambles.mjs +95 -0
- package/bin/run-migrations.mjs +1 -10
- package/bin/semantic-search.mjs +11 -19
- package/bin/session-start-launcher.mjs +102 -100
- package/bin/simplify-classify.cjs +38 -17
- package/dist/src/cli/commands/daemon.js +38 -11
- package/dist/src/cli/commands/doctor-checks-coverage-truth.js +136 -0
- package/dist/src/cli/commands/doctor-checks-memory-access.js +244 -5
- package/dist/src/cli/commands/doctor-checks-memory.js +13 -18
- package/dist/src/cli/commands/doctor-checks-version-skew.js +94 -0
- package/dist/src/cli/commands/doctor-checks-writers-audit.js +170 -0
- package/dist/src/cli/commands/doctor-embedding-hygiene.js +3 -15
- package/dist/src/cli/commands/doctor-fixes.js +30 -0
- package/dist/src/cli/commands/doctor-registry.js +14 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/embeddings.js +17 -22
- package/dist/src/cli/commands/memory.js +54 -75
- package/dist/src/cli/embeddings/persistent-cache.js +44 -83
- package/dist/src/cli/init/claudemd-generator.js +4 -0
- package/dist/src/cli/init/moflo-init.js +40 -0
- package/dist/src/cli/mcp-tools/memory-tools.js +177 -32
- package/dist/src/cli/memory/bridge-core.js +256 -30
- package/dist/src/cli/memory/bridge-entries.js +76 -8
- package/dist/src/cli/memory/controller-registry.js +7 -2
- package/dist/src/cli/memory/controllers/batch-operations.js +5 -1
- package/dist/src/cli/memory/controllers/hierarchical-memory.js +7 -2
- package/dist/src/cli/memory/controllers/mutation-guard.js +22 -2
- package/dist/src/cli/memory/daemon-backend.js +400 -0
- package/dist/src/cli/memory/daemon-write-client.js +192 -15
- package/dist/src/cli/memory/database-provider.js +57 -40
- package/dist/src/cli/memory/hnsw-persistence.js +6 -8
- package/dist/src/cli/memory/index.js +0 -1
- package/dist/src/cli/memory/memory-bridge.js +40 -8
- package/dist/src/cli/memory/memory-initializer.js +286 -220
- package/dist/src/cli/memory/rvf-migration.js +25 -11
- package/dist/src/cli/memory/sqlite-backend.js +573 -0
- package/dist/src/cli/memory/suppress-sqlite-warning.js +49 -0
- package/dist/src/cli/services/cherry-pick-learnings.js +32 -21
- package/dist/src/cli/services/daemon-dashboard.js +13 -1
- package/dist/src/cli/services/daemon-lock.js +58 -1
- package/dist/src/cli/services/daemon-memory-rpc.js +245 -10
- package/dist/src/cli/services/embeddings-migration.js +9 -12
- package/dist/src/cli/services/ephemeral-namespace-purge.js +6 -11
- package/dist/src/cli/services/learning-service.js +12 -20
- package/dist/src/cli/services/project-root.js +69 -9
- package/dist/src/cli/services/soft-delete-purge.js +6 -11
- package/dist/src/cli/services/sqljs-migration-store.js +4 -1
- package/dist/src/cli/services/subagent-bootstrap.js +1 -1
- package/dist/src/cli/shared/events/event-store.js +26 -55
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -4
- package/dist/src/cli/memory/sqljs-backend.js +0 -643
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Daemon-side memory DB factory — TS twin of `bin/lib/get-backend.mjs`.
|
|
3
|
+
*
|
|
4
|
+
* Returns a handle whose surface matches the sql.js `Database` type that the
|
|
5
|
+
* controller registry + bridge code expect (prepare → Statement, run, exec,
|
|
6
|
+
* close, plus a `save()` that maps to the engine's preferred persistence).
|
|
7
|
+
* Currently always returns a node:sqlite-backed adapter — Phase 4 (#1083)
|
|
8
|
+
* flipped the SQLite default; Phase 5 (#1084) deletes the remaining sql.js
|
|
9
|
+
* paths in the bridge + memory-initializer.
|
|
10
|
+
*
|
|
11
|
+
* The sql.js Statement API the bridge code relies on:
|
|
12
|
+
* - db.prepare(sql) → stmt
|
|
13
|
+
* - db.run(sql, params?)
|
|
14
|
+
* - db.exec(sql) → [{ columns, values }]
|
|
15
|
+
* - db.close()
|
|
16
|
+
* - stmt.bind(params)
|
|
17
|
+
* - stmt.step() → boolean
|
|
18
|
+
* - stmt.getAsObject() → row object
|
|
19
|
+
* - stmt.run(params?) → boolean
|
|
20
|
+
* - stmt.free()
|
|
21
|
+
*
|
|
22
|
+
* node:sqlite's `StatementSync` is stateless (each call takes params), so we
|
|
23
|
+
* shim a stateful wrapper via `stmt.iterate(...)` opened on first `step()`.
|
|
24
|
+
* This is the same shape implemented in bin/lib/get-backend.mjs — keep the
|
|
25
|
+
* two in lockstep until Phase 5 collapses them.
|
|
26
|
+
*
|
|
27
|
+
* @module v3/memory/daemon-backend
|
|
28
|
+
*/
|
|
29
|
+
// MUST come before `import 'node:sqlite'` below — that module fires
|
|
30
|
+
// `ExperimentalWarning: SQLite is an experimental feature` exactly once per
|
|
31
|
+
// process on first load. Once it fires, there's no way to scrub it from
|
|
32
|
+
// stderr, and consumer-smoke's 200-char stderr tails get filled with it
|
|
33
|
+
// (hiding the real error message; #1098).
|
|
34
|
+
import './suppress-sqlite-warning.js';
|
|
35
|
+
import { existsSync, mkdirSync, readFileSync } from 'node:fs';
|
|
36
|
+
import { dirname } from 'node:path';
|
|
37
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
38
|
+
/**
|
|
39
|
+
* Coerce a caller-supplied parameter set into node:sqlite's `SQLInputValue[]`
|
|
40
|
+
* shape. Callers pass `any` (sql.js parity) so we just pass through after
|
|
41
|
+
* normalising null/undefined/array.
|
|
42
|
+
*/
|
|
43
|
+
function toParamsArray(params) {
|
|
44
|
+
if (params === undefined || params === null)
|
|
45
|
+
return [];
|
|
46
|
+
if (Array.isArray(params))
|
|
47
|
+
return params;
|
|
48
|
+
// sql.js's `Statement.bind(obj)` named-param shape isn't reachable from
|
|
49
|
+
// moflo's bridge code today — every caller passes an array. Tolerate
|
|
50
|
+
// anyway by wrapping the lone value.
|
|
51
|
+
return [params];
|
|
52
|
+
}
|
|
53
|
+
function wrapStatement(stmt) {
|
|
54
|
+
let pendingParams = [];
|
|
55
|
+
let iter = null;
|
|
56
|
+
let currentRow = null;
|
|
57
|
+
let columnNamesCache = null;
|
|
58
|
+
const ensureIter = () => {
|
|
59
|
+
if (!iter) {
|
|
60
|
+
iter = (pendingParams.length > 0
|
|
61
|
+
? stmt.iterate(...pendingParams)
|
|
62
|
+
: stmt.iterate());
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
bind(params) {
|
|
67
|
+
pendingParams = toParamsArray(params);
|
|
68
|
+
iter = null;
|
|
69
|
+
currentRow = null;
|
|
70
|
+
return true;
|
|
71
|
+
},
|
|
72
|
+
step() {
|
|
73
|
+
ensureIter();
|
|
74
|
+
const next = iter.next();
|
|
75
|
+
if (next.done) {
|
|
76
|
+
currentRow = null;
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
currentRow = next.value;
|
|
80
|
+
return true;
|
|
81
|
+
},
|
|
82
|
+
getAsObject(params) {
|
|
83
|
+
// sql.js semantics: with params it's a one-shot bind+step+return;
|
|
84
|
+
// without params it returns whatever the last step() materialised. The
|
|
85
|
+
// bridge uses both shapes (one-shot in bridge-entries.ts:bridgeGetEntry,
|
|
86
|
+
// iterator in list/search). Returning {} from the one-shot form when
|
|
87
|
+
// there's no row is correct (caller checks for nullish primary key).
|
|
88
|
+
if (params !== undefined) {
|
|
89
|
+
pendingParams = toParamsArray(params);
|
|
90
|
+
iter = null;
|
|
91
|
+
ensureIter();
|
|
92
|
+
const next = iter.next();
|
|
93
|
+
if (next.done) {
|
|
94
|
+
currentRow = null;
|
|
95
|
+
return {};
|
|
96
|
+
}
|
|
97
|
+
currentRow = next.value;
|
|
98
|
+
return currentRow;
|
|
99
|
+
}
|
|
100
|
+
return currentRow ?? {};
|
|
101
|
+
},
|
|
102
|
+
get(params) {
|
|
103
|
+
// sql.js `Statement.get()` semantics:
|
|
104
|
+
// - With params: one-shot bind+step, returns positional values of the
|
|
105
|
+
// first row (or [] if no rows).
|
|
106
|
+
// - Without params: returns positional values of the CURRENT row —
|
|
107
|
+
// the row that the most-recent `step()` landed on. Callers use it
|
|
108
|
+
// in a `while (stmt.step()) { const row = stmt.get(); ... }` loop;
|
|
109
|
+
// calling iter.next() again here would skip every other row.
|
|
110
|
+
if (params !== undefined) {
|
|
111
|
+
pendingParams = toParamsArray(params);
|
|
112
|
+
iter = null;
|
|
113
|
+
ensureIter();
|
|
114
|
+
const next = iter.next();
|
|
115
|
+
if (next.done) {
|
|
116
|
+
currentRow = null;
|
|
117
|
+
return [];
|
|
118
|
+
}
|
|
119
|
+
currentRow = next.value;
|
|
120
|
+
}
|
|
121
|
+
if (!currentRow)
|
|
122
|
+
return [];
|
|
123
|
+
const cols = columnNamesCache ?? Object.keys(currentRow);
|
|
124
|
+
columnNamesCache = cols;
|
|
125
|
+
return cols.map((c) => currentRow[c]);
|
|
126
|
+
},
|
|
127
|
+
getColumnNames() {
|
|
128
|
+
if (columnNamesCache)
|
|
129
|
+
return columnNamesCache;
|
|
130
|
+
// Force one step to materialise a row so column names are knowable.
|
|
131
|
+
ensureIter();
|
|
132
|
+
const next = iter.next();
|
|
133
|
+
if (next.done) {
|
|
134
|
+
columnNamesCache = [];
|
|
135
|
+
currentRow = null;
|
|
136
|
+
return [];
|
|
137
|
+
}
|
|
138
|
+
currentRow = next.value;
|
|
139
|
+
columnNamesCache = Object.keys(currentRow);
|
|
140
|
+
return columnNamesCache;
|
|
141
|
+
},
|
|
142
|
+
run(params) {
|
|
143
|
+
const arr = toParamsArray(params);
|
|
144
|
+
if (arr.length > 0)
|
|
145
|
+
stmt.run(...arr);
|
|
146
|
+
else
|
|
147
|
+
stmt.run();
|
|
148
|
+
return true;
|
|
149
|
+
},
|
|
150
|
+
reset() {
|
|
151
|
+
iter = null;
|
|
152
|
+
currentRow = null;
|
|
153
|
+
},
|
|
154
|
+
free() {
|
|
155
|
+
iter = null;
|
|
156
|
+
currentRow = null;
|
|
157
|
+
pendingParams = [];
|
|
158
|
+
columnNamesCache = null;
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Per-process dedupe of network-FS warnings — emit once per (dbPath, process).
|
|
164
|
+
* Matches the JS twin's `_networkFsWarnedPaths` set so a session that opens
|
|
165
|
+
* both the daemon adapter and a bin/ writer on the same path only logs once.
|
|
166
|
+
*/
|
|
167
|
+
const _networkFsWarnedPaths = new Set();
|
|
168
|
+
/**
|
|
169
|
+
* Read `journal_mode` back after we requested WAL. If the engine returned a
|
|
170
|
+
* different mode (`delete`, `truncate`, `persist`, `memory`, `off`), the
|
|
171
|
+
* underlying filesystem doesn't support WAL's shared-memory sidecar — a
|
|
172
|
+
* strong signal that POSIX advisory locks are also unreliable. Surface a
|
|
173
|
+
* one-line stderr warning naming the path so the user knows to move the
|
|
174
|
+
* project off the network mount. Deduped per (path, process).
|
|
175
|
+
*
|
|
176
|
+
* Twin: `bin/lib/get-backend.mjs:warnIfNotWal`. Must stay in lockstep until
|
|
177
|
+
* Phase 5 (#1084) extracts a shared module.
|
|
178
|
+
*/
|
|
179
|
+
function warnIfNotWal(db, dbPath) {
|
|
180
|
+
if (_networkFsWarnedPaths.has(dbPath))
|
|
181
|
+
return;
|
|
182
|
+
let mode;
|
|
183
|
+
try {
|
|
184
|
+
const stmt = db.prepare('PRAGMA journal_mode');
|
|
185
|
+
const row = stmt.get();
|
|
186
|
+
mode = String(row?.journal_mode ?? '').toLowerCase();
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
if (mode && mode !== 'wal') {
|
|
192
|
+
_networkFsWarnedPaths.add(dbPath);
|
|
193
|
+
process.stderr.write(`[moflo] WARNING: SQLite journal_mode=${mode} on ${dbPath} (WAL not active). ` +
|
|
194
|
+
`If this directory is on NFS/SMB or another network filesystem, POSIX ` +
|
|
195
|
+
`advisory locks are unreliable and concurrent moflo processes can corrupt ` +
|
|
196
|
+
`the database. Move the project to a local disk to restore multi-process safety.\n`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/** @internal — test hook only (resets the dedupe set). */
|
|
200
|
+
export function _resetDaemonNetworkFsWarnings() {
|
|
201
|
+
_networkFsWarnedPaths.clear();
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Heuristic: SQL is multi-statement if there's a `;` followed by anything
|
|
205
|
+
* substantive. node:sqlite's `db.prepare()` does NOT throw on multi-statement
|
|
206
|
+
* input — it silently parses only the first statement and discards the rest,
|
|
207
|
+
* which is the worst possible failure mode for DDL batches like
|
|
208
|
+
* `CREATE TABLE a; CREATE INDEX i; CREATE TABLE b;`. Detect explicitly and
|
|
209
|
+
* route multi-stmt SQL to `db.exec()` (which runs every statement).
|
|
210
|
+
*
|
|
211
|
+
* The bridge + controller schema strings don't embed literal `;` inside
|
|
212
|
+
* string literals, so the naive index-of check is sound. If that ever
|
|
213
|
+
* changes, this needs a real tokeniser.
|
|
214
|
+
*/
|
|
215
|
+
function isMultiStatement(sql) {
|
|
216
|
+
const trimmed = sql.trimEnd();
|
|
217
|
+
const semi = trimmed.indexOf(';');
|
|
218
|
+
if (semi === -1)
|
|
219
|
+
return false;
|
|
220
|
+
return /\S/.test(trimmed.slice(semi + 1));
|
|
221
|
+
}
|
|
222
|
+
function execAsRowsNodeSqlite(db, sql, params) {
|
|
223
|
+
// Multi-statement DDL: route to db.exec() so every statement runs.
|
|
224
|
+
// (sql.js's exec() runs every statement and returns row sets from any
|
|
225
|
+
// that produce rows; the bridge code only reads [0]?.values, so DDL
|
|
226
|
+
// batches correctly return []. Match that contract.)
|
|
227
|
+
if (isMultiStatement(sql)) {
|
|
228
|
+
db.exec(sql);
|
|
229
|
+
return [];
|
|
230
|
+
}
|
|
231
|
+
let stmt;
|
|
232
|
+
try {
|
|
233
|
+
stmt = db.prepare(sql);
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
// Last resort for any single-statement SQL that prepare rejects.
|
|
237
|
+
db.exec(sql);
|
|
238
|
+
return [];
|
|
239
|
+
}
|
|
240
|
+
const args = toParamsArray(params);
|
|
241
|
+
const rows = (args.length > 0 ? stmt.all(...args) : stmt.all());
|
|
242
|
+
if (rows.length === 0)
|
|
243
|
+
return [];
|
|
244
|
+
const columns = Object.keys(rows[0]);
|
|
245
|
+
const values = rows.map((r) => columns.map((c) => r[c]));
|
|
246
|
+
return [{ columns, values }];
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Open the daemon's memory DB handle. Always returns a node:sqlite-backed
|
|
250
|
+
* adapter shaped like sql.js's Database so the existing bridge + controller
|
|
251
|
+
* surface keeps working.
|
|
252
|
+
*
|
|
253
|
+
* @param dbPath disk path or `:memory:`
|
|
254
|
+
*/
|
|
255
|
+
export function openDaemonDatabase(dbPath) {
|
|
256
|
+
// node:sqlite opens forgivingly even on non-SQLite files. Keep parity with
|
|
257
|
+
// openSqlJsDatabase's "create if missing" semantic — DatabaseSync handles
|
|
258
|
+
// file creation for us, BUT does NOT auto-create parent directories. The
|
|
259
|
+
// bridge's first-init path commonly lands on a path whose parent .moflo/
|
|
260
|
+
// doesn't exist yet (fresh consumer install, test fixtures with temp
|
|
261
|
+
// project roots) — without the mkdir below, DatabaseSync throws ENOENT,
|
|
262
|
+
// the controller-registry sets mofloDb=null, and the bridge silently
|
|
263
|
+
// falls back to a raw-sql.js write rooted at process.cwd() (catastrophic
|
|
264
|
+
// path drift bug; #1057 was about exactly this class of issue).
|
|
265
|
+
if (dbPath !== ':memory:') {
|
|
266
|
+
try {
|
|
267
|
+
mkdirSync(dirname(dbPath), { recursive: true });
|
|
268
|
+
}
|
|
269
|
+
catch { /* tolerate — DatabaseSync's ENOENT below is the real signal */ }
|
|
270
|
+
}
|
|
271
|
+
const db = new DatabaseSync(dbPath);
|
|
272
|
+
if (dbPath !== ':memory:') {
|
|
273
|
+
try {
|
|
274
|
+
// busy_timeout MUST be set BEFORE journal_mode=WAL, because the WAL
|
|
275
|
+
// pragma briefly takes an EXCLUSIVE lock and concurrent openers race
|
|
276
|
+
// on it. Without busy_timeout in place, parallel doctor probes /
|
|
277
|
+
// bridge initializations / indexer subprocess opens hit "database is
|
|
278
|
+
// locked" and the bridge tears down (CI #1097). Order matters:
|
|
279
|
+
// 1. busy_timeout — gives every subsequent pragma a retry budget
|
|
280
|
+
// 2. journal_mode = WAL — needs the budget on contention
|
|
281
|
+
// 3. synchronous — purely advisory, can come anytime
|
|
282
|
+
//
|
|
283
|
+
// Budget: 15000ms. The consumer-smoke harness exposes the realistic
|
|
284
|
+
// worst case — a background indexer subprocess opens its own write
|
|
285
|
+
// connection right after `npm install` and walks the entire consumer
|
|
286
|
+
// tree (hundreds of guidance/skill files). The whole-tree first-pass
|
|
287
|
+
// can hold a RESERVED/EXCLUSIVE lock for 5–8s while the doctor
|
|
288
|
+
// foreground probe races against it. 5000ms was the original Phase 4
|
|
289
|
+
// value and ran the budget out under that exact load on Windows CI
|
|
290
|
+
// (#1098); 15000ms gives the indexer's full first-pass time to finish
|
|
291
|
+
// before doctor's probe gives up. The price of being wrong-high here
|
|
292
|
+
// is one slow probe per session, not lost data.
|
|
293
|
+
db.exec('PRAGMA busy_timeout = 15000');
|
|
294
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
295
|
+
db.exec('PRAGMA synchronous = NORMAL');
|
|
296
|
+
// The daemon is the process most exposed to network-FS edge cases
|
|
297
|
+
// (long-lived MCP server, ~30s of writes per indexer pass). NFS/SMB
|
|
298
|
+
// mounts silently fall back from WAL to a rollback journal — surface
|
|
299
|
+
// a one-line warning so the user knows to move the project off the
|
|
300
|
+
// network mount. Mirrors `bin/lib/get-backend.mjs:warnIfNotWal`.
|
|
301
|
+
warnIfNotWal(db, dbPath);
|
|
302
|
+
}
|
|
303
|
+
catch (err) {
|
|
304
|
+
try {
|
|
305
|
+
db.close();
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
/* handle already dead */
|
|
309
|
+
}
|
|
310
|
+
throw err;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
// sql.js's `db.run(sql, params?)` and `prepare/run` share state; node:sqlite
|
|
314
|
+
// requires fresh statements per call. Cache prepared statements keyed by SQL
|
|
315
|
+
// text so the indexer-equivalent tight write loops don't churn the compiler.
|
|
316
|
+
const runStmtCache = new Map();
|
|
317
|
+
let changesStmt = null;
|
|
318
|
+
return {
|
|
319
|
+
kind: 'node-sqlite',
|
|
320
|
+
_raw: db,
|
|
321
|
+
prepare(sql) {
|
|
322
|
+
return wrapStatement(db.prepare(sql));
|
|
323
|
+
},
|
|
324
|
+
run(sql, params) {
|
|
325
|
+
const arr = toParamsArray(params);
|
|
326
|
+
if (arr.length > 0) {
|
|
327
|
+
let s = runStmtCache.get(sql);
|
|
328
|
+
if (!s) {
|
|
329
|
+
s = db.prepare(sql);
|
|
330
|
+
runStmtCache.set(sql, s);
|
|
331
|
+
}
|
|
332
|
+
s.run(...arr);
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
db.exec(sql);
|
|
336
|
+
}
|
|
337
|
+
return undefined;
|
|
338
|
+
},
|
|
339
|
+
exec(sql, params) {
|
|
340
|
+
return execAsRowsNodeSqlite(db, sql, params);
|
|
341
|
+
},
|
|
342
|
+
getRowsModified() {
|
|
343
|
+
if (!changesStmt)
|
|
344
|
+
changesStmt = db.prepare('SELECT changes() AS c');
|
|
345
|
+
const row = changesStmt.get();
|
|
346
|
+
const c = row?.c ?? 0;
|
|
347
|
+
return typeof c === 'bigint' ? Number(c) : c;
|
|
348
|
+
},
|
|
349
|
+
save() {
|
|
350
|
+
// node:sqlite persists incrementally via WAL — no-op. The shape exists
|
|
351
|
+
// so bridge-core's persistBridgeDb can call it unconditionally during
|
|
352
|
+
// the Phase 4/5 transition window. Once everything routes through this
|
|
353
|
+
// adapter, the explicit persist call becomes dead code (Phase 5).
|
|
354
|
+
},
|
|
355
|
+
export() {
|
|
356
|
+
// Bridge-core's old persist path used `db.export()` + atomicWriteFileSync.
|
|
357
|
+
// node:sqlite ships a `serialize()` that returns the same shape so the
|
|
358
|
+
// few callers that still need a buffer (e.g. tests, backup tooling) work.
|
|
359
|
+
// The bridge-core persist call itself is being switched to `save()` so
|
|
360
|
+
// this exists only as a safety net during the migration.
|
|
361
|
+
const buf = db.prepare('SELECT 1').all();
|
|
362
|
+
void buf;
|
|
363
|
+
// Real serialize. node:sqlite added `DatabaseSync.prototype.serialize()`
|
|
364
|
+
// in Node 22; the TS type for it landed later, so cast through the
|
|
365
|
+
// engine's runtime API.
|
|
366
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
367
|
+
const ser = db.serialize?.();
|
|
368
|
+
if (ser instanceof Uint8Array)
|
|
369
|
+
return ser;
|
|
370
|
+
if (ser && typeof ser === 'object' && 'buffer' in ser)
|
|
371
|
+
return new Uint8Array(ser);
|
|
372
|
+
// Last resort: read the file off disk. The caller knows the path; we
|
|
373
|
+
// don't, so this branch should never fire under normal flow.
|
|
374
|
+
return new Uint8Array();
|
|
375
|
+
},
|
|
376
|
+
close() {
|
|
377
|
+
runStmtCache.clear();
|
|
378
|
+
changesStmt = null;
|
|
379
|
+
db.close();
|
|
380
|
+
},
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Seed an empty daemon DB from an existing file on disk. Equivalent to
|
|
385
|
+
* sql.js's `new SQL.Database(readFileSync(path))` round-trip — node:sqlite
|
|
386
|
+
* opens the path directly so this is just a wrapper that errors when the
|
|
387
|
+
* file doesn't exist (existing callers expected the sql.js behaviour).
|
|
388
|
+
*/
|
|
389
|
+
export function openDaemonDatabaseFromFile(dbPath) {
|
|
390
|
+
if (dbPath !== ':memory:' && !existsSync(dbPath)) {
|
|
391
|
+
throw new Error(`Database file not found: ${dbPath}`);
|
|
392
|
+
}
|
|
393
|
+
// Touch readFileSync so callers that previously expected eager I/O still
|
|
394
|
+
// observe the same failure shape (e.g. EACCES errors fire here, not
|
|
395
|
+
// lazily on first query). node:sqlite would lazy-error otherwise.
|
|
396
|
+
if (dbPath !== ':memory:')
|
|
397
|
+
readFileSync(dbPath, { flag: 'r' });
|
|
398
|
+
return openDaemonDatabase(dbPath);
|
|
399
|
+
}
|
|
400
|
+
//# sourceMappingURL=daemon-backend.js.map
|
|
@@ -1,22 +1,38 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Daemon
|
|
2
|
+
* Daemon RPC client (#981 / #984 / #1058 — single-writer architecture and
|
|
3
|
+
* its read-side symmetry).
|
|
3
4
|
*
|
|
4
|
-
* HTTP client for the `POST /api/memory/{store,delete,batch}`
|
|
5
|
-
*
|
|
6
|
-
* route their `.moflo/moflo.db`
|
|
7
|
-
* authoritative sql.js handle. Avoids the
|
|
5
|
+
* HTTP client for the `POST /api/memory/{store,delete,batch,get,search,list}`
|
|
6
|
+
* RPC added by Stories #983 and #1058. Lets short-lived CLI processes and
|
|
7
|
+
* the long-lived MCP server route their `.moflo/moflo.db` operations through
|
|
8
|
+
* the daemon, which owns the authoritative sql.js handle. Avoids the
|
|
9
|
+
* multi-process write clobber from #981 AND the stale read snapshot from
|
|
10
|
+
* #1058 (sql.js never re-reads disk after init).
|
|
8
11
|
*
|
|
9
12
|
* Contract — every function in this module:
|
|
10
|
-
* - Never throws.
|
|
13
|
+
* - Never throws. Outcomes are reported via the result envelope.
|
|
11
14
|
* - Returns within ≤100ms even if the daemon is dead/slow (HTTP timeout).
|
|
12
|
-
* - Caches daemon health for 5s to keep the hot
|
|
15
|
+
* - Caches daemon health for 5s to keep the hot path cheap.
|
|
13
16
|
* - Short-circuits without HTTP when:
|
|
14
17
|
* (a) `process.env.MOFLO_IS_DAEMON === '1'` (daemon's own process)
|
|
15
18
|
* (b) `moflo.yaml` has `daemon.auto_start: false`
|
|
16
19
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* `
|
|
20
|
+
* Failure-shape contract (#1101 — surface 4xx as a real error):
|
|
21
|
+
* - 4xx response → `routed: true, ok: false, error: <msg>` (writes)
|
|
22
|
+
* → `routed: true, error: <msg>` (reads)
|
|
23
|
+
* → caller PROPAGATES the error; does NOT fall back.
|
|
24
|
+
* - 5xx, 503, timeout, → `routed: false`
|
|
25
|
+
* ECONNREFUSED, malformed JSON, → caller falls back to bridge-direct.
|
|
26
|
+
* socket destroyed mid-stream
|
|
27
|
+
*
|
|
28
|
+
* The 4xx codes only fire on daemon-side payload validation (see
|
|
29
|
+
* daemon-memory-rpc.ts). Bridge-direct has the same validation and would
|
|
30
|
+
* fail the same way — falling back silently loses the daemon's actionable
|
|
31
|
+
* error message. 5xx and transport faults are transient/daemon-side bugs;
|
|
32
|
+
* bridge-direct is the right next step.
|
|
33
|
+
*
|
|
34
|
+
* Naming note: the module is named `daemon-write-client` for compat with
|
|
35
|
+
* existing importers, but as of #1058 it also covers reads.
|
|
20
36
|
*
|
|
21
37
|
* @module cli/memory/daemon-write-client
|
|
22
38
|
*/
|
|
@@ -136,6 +152,7 @@ export async function tryDaemonStore(opts) {
|
|
|
136
152
|
value: opts.value,
|
|
137
153
|
tags: opts.tags,
|
|
138
154
|
ttl: opts.ttl,
|
|
155
|
+
metadata: opts.metadata,
|
|
139
156
|
});
|
|
140
157
|
}
|
|
141
158
|
/**
|
|
@@ -150,9 +167,90 @@ export async function tryDaemonDelete(opts) {
|
|
|
150
167
|
key: opts.key,
|
|
151
168
|
});
|
|
152
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Route a single-entry retrieve through the daemon (#1058). Returns
|
|
172
|
+
* `{ routed: false }` if the daemon is unavailable; otherwise
|
|
173
|
+
* `{ routed: true, data: { found, entry? } }`. The `entry` field is the
|
|
174
|
+
* same shape as `getEntry`'s in-process return.
|
|
175
|
+
*/
|
|
176
|
+
export async function tryDaemonGet(opts) {
|
|
177
|
+
if (!(await isDaemonAvailable()))
|
|
178
|
+
return { routed: false };
|
|
179
|
+
return postReadJson('/api/memory/get', { namespace: opts.namespace, key: opts.key }, (data) => ({
|
|
180
|
+
found: !!data?.found,
|
|
181
|
+
entry: data?.entry,
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Route a semantic search through the daemon (#1058).
|
|
186
|
+
*/
|
|
187
|
+
export async function tryDaemonSearch(opts) {
|
|
188
|
+
if (!(await isDaemonAvailable()))
|
|
189
|
+
return { routed: false };
|
|
190
|
+
return postReadJson('/api/memory/search', {
|
|
191
|
+
query: opts.query,
|
|
192
|
+
namespace: opts.namespace,
|
|
193
|
+
limit: opts.limit,
|
|
194
|
+
threshold: opts.threshold,
|
|
195
|
+
}, (data) => ({
|
|
196
|
+
results: Array.isArray(data?.results) ? data.results : [],
|
|
197
|
+
searchTime: typeof data?.searchTime === 'number' ? data.searchTime : undefined,
|
|
198
|
+
}));
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Route a paginated list through the daemon (#1058).
|
|
202
|
+
*/
|
|
203
|
+
export async function tryDaemonList(opts) {
|
|
204
|
+
if (!(await isDaemonAvailable()))
|
|
205
|
+
return { routed: false };
|
|
206
|
+
return postReadJson('/api/memory/list', {
|
|
207
|
+
namespace: opts.namespace,
|
|
208
|
+
limit: opts.limit,
|
|
209
|
+
offset: opts.offset,
|
|
210
|
+
}, (data) => ({
|
|
211
|
+
entries: Array.isArray(data?.entries) ? data.entries : [],
|
|
212
|
+
total: typeof data?.total === 'number' ? data.total : 0,
|
|
213
|
+
}));
|
|
214
|
+
}
|
|
153
215
|
// ============================================================================
|
|
154
216
|
// Internal HTTP poster — never throws, bounded timeout
|
|
155
217
|
// ============================================================================
|
|
218
|
+
/**
|
|
219
|
+
* Extract a human-readable error message from a daemon 4xx response body.
|
|
220
|
+
* Prefers `message` (the daemon's specific reason — e.g. "invalid namespace"),
|
|
221
|
+
* falls back to `error` (the daemon's error category), then to a generic
|
|
222
|
+
* status-code string when the body is non-JSON.
|
|
223
|
+
*/
|
|
224
|
+
function parse4xxError(buf, status) {
|
|
225
|
+
try {
|
|
226
|
+
const data = JSON.parse(buf);
|
|
227
|
+
const detail = typeof data?.message === 'string' ? data.message
|
|
228
|
+
: typeof data?.error === 'string' ? data.error
|
|
229
|
+
: undefined;
|
|
230
|
+
if (detail)
|
|
231
|
+
return detail;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
// Non-JSON 4xx body — fall through to the generic message.
|
|
235
|
+
}
|
|
236
|
+
return `daemon returned ${status}`;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Narrow a parsed JSON value to the `{ dimensions, model }` embedding-response
|
|
240
|
+
* shape (#1065). Returns `undefined` when the field is missing or malformed —
|
|
241
|
+
* a malformed field is treated as "no embedding info" rather than failing the
|
|
242
|
+
* whole response, so an older daemon that hasn't been updated still works.
|
|
243
|
+
*/
|
|
244
|
+
function parseEmbeddingField(value) {
|
|
245
|
+
if (!value || typeof value !== 'object')
|
|
246
|
+
return undefined;
|
|
247
|
+
const v = value;
|
|
248
|
+
if (typeof v.dimensions !== 'number' || !Number.isFinite(v.dimensions))
|
|
249
|
+
return undefined;
|
|
250
|
+
if (typeof v.model !== 'string' || v.model.length === 0)
|
|
251
|
+
return undefined;
|
|
252
|
+
return { dimensions: v.dimensions, model: v.model };
|
|
253
|
+
}
|
|
156
254
|
function postJson(path, body) {
|
|
157
255
|
return new Promise((resolve) => {
|
|
158
256
|
let done = false;
|
|
@@ -183,15 +281,23 @@ function postJson(path, body) {
|
|
|
183
281
|
res.setEncoding('utf8');
|
|
184
282
|
res.on('data', (chunk) => { buf += chunk; });
|
|
185
283
|
res.on('end', () => {
|
|
186
|
-
//
|
|
187
|
-
// caller
|
|
188
|
-
//
|
|
189
|
-
//
|
|
284
|
+
// #1101 — per-shape failure contract:
|
|
285
|
+
// 2xx → routed:true, ok:true (caller uses data)
|
|
286
|
+
// 4xx → routed:true, ok:false (caller propagates daemon error)
|
|
287
|
+
// 5xx → routed:false (caller falls back to bridge)
|
|
288
|
+
// parse fail → routed:false (fall back)
|
|
190
289
|
const status = res.statusCode ?? 0;
|
|
191
|
-
if (status
|
|
290
|
+
if (status >= 500 || status < 200) {
|
|
192
291
|
finish({ routed: false });
|
|
193
292
|
return;
|
|
194
293
|
}
|
|
294
|
+
if (status >= 400) {
|
|
295
|
+
// Daemon validated the payload and rejected it. Bridge-direct
|
|
296
|
+
// has the same validation; falling back loses the actionable
|
|
297
|
+
// error. Surface it to the caller instead.
|
|
298
|
+
finish({ routed: true, ok: false, error: parse4xxError(buf, status) });
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
195
301
|
try {
|
|
196
302
|
const data = JSON.parse(buf);
|
|
197
303
|
finish({
|
|
@@ -199,6 +305,7 @@ function postJson(path, body) {
|
|
|
199
305
|
ok: !!data?.ok,
|
|
200
306
|
id: typeof data?.id === 'string' ? data.id : undefined,
|
|
201
307
|
deleted: typeof data?.deleted === 'boolean' ? data.deleted : undefined,
|
|
308
|
+
embedding: parseEmbeddingField(data?.embedding),
|
|
202
309
|
error: typeof data?.error === 'string' ? data.error : undefined,
|
|
203
310
|
});
|
|
204
311
|
}
|
|
@@ -214,4 +321,74 @@ function postJson(path, body) {
|
|
|
214
321
|
req.end();
|
|
215
322
|
});
|
|
216
323
|
}
|
|
324
|
+
/**
|
|
325
|
+
* Generic JSON POST that returns a daemon-read envelope. Same transport
|
|
326
|
+
* guarantees as `postJson`: never throws, bounded timeout, invalidates health
|
|
327
|
+
* cache on routed-failure.
|
|
328
|
+
*
|
|
329
|
+
* The `shape` callback maps the daemon's parsed JSON payload to the typed
|
|
330
|
+
* data shape the caller expects. Returning `null` from `shape` (or a parse
|
|
331
|
+
* failure) downgrades to `{ routed: false }` so the caller falls back.
|
|
332
|
+
*/
|
|
333
|
+
function postReadJson(path, body, shape) {
|
|
334
|
+
return new Promise((resolve) => {
|
|
335
|
+
let done = false;
|
|
336
|
+
const finish = (result) => {
|
|
337
|
+
if (done)
|
|
338
|
+
return;
|
|
339
|
+
done = true;
|
|
340
|
+
if (result.routed === false)
|
|
341
|
+
healthCache = null;
|
|
342
|
+
resolve(result);
|
|
343
|
+
};
|
|
344
|
+
const payload = JSON.stringify(body);
|
|
345
|
+
const req = http.request({
|
|
346
|
+
host: '127.0.0.1',
|
|
347
|
+
port: getDaemonPort(),
|
|
348
|
+
path,
|
|
349
|
+
method: 'POST',
|
|
350
|
+
timeout: DAEMON_HTTP_TIMEOUT_MS,
|
|
351
|
+
headers: {
|
|
352
|
+
'Content-Type': 'application/json',
|
|
353
|
+
'Content-Length': Buffer.byteLength(payload),
|
|
354
|
+
},
|
|
355
|
+
}, (res) => {
|
|
356
|
+
let buf = '';
|
|
357
|
+
res.setEncoding('utf8');
|
|
358
|
+
res.on('data', (chunk) => { buf += chunk; });
|
|
359
|
+
res.on('end', () => {
|
|
360
|
+
// #1101 — mirror postJson contract for reads:
|
|
361
|
+
// 2xx → routed:true with shaped data
|
|
362
|
+
// 4xx → routed:true with error (no data) — caller propagates
|
|
363
|
+
// 5xx → routed:false (caller falls back)
|
|
364
|
+
const status = res.statusCode ?? 0;
|
|
365
|
+
if (status >= 500 || status < 200) {
|
|
366
|
+
finish({ routed: false });
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (status >= 400) {
|
|
370
|
+
finish({ routed: true, error: parse4xxError(buf, status) });
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
try {
|
|
374
|
+
const parsed = JSON.parse(buf);
|
|
375
|
+
const shaped = shape(parsed);
|
|
376
|
+
if (shaped === null) {
|
|
377
|
+
finish({ routed: false });
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
finish({ routed: true, data: shaped });
|
|
381
|
+
}
|
|
382
|
+
catch {
|
|
383
|
+
finish({ routed: false });
|
|
384
|
+
}
|
|
385
|
+
});
|
|
386
|
+
res.on('error', () => finish({ routed: false }));
|
|
387
|
+
});
|
|
388
|
+
req.on('error', () => finish({ routed: false }));
|
|
389
|
+
req.on('timeout', () => { req.destroy(); finish({ routed: false }); });
|
|
390
|
+
req.write(payload);
|
|
391
|
+
req.end();
|
|
392
|
+
});
|
|
393
|
+
}
|
|
217
394
|
//# sourceMappingURL=daemon-write-client.js.map
|