moflo 4.9.36 → 4.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-agent-rules.md +12 -0
- package/.claude/guidance/shipped/moflo-memory-protocol.md +34 -0
- package/.claude/guidance/shipped/moflo-memorydb-maintenance.md +22 -11
- package/.claude/guidance/shipped/moflo-root-cause-discipline.md +47 -0
- package/.claude/guidance/shipped/moflo-subagents.md +4 -0
- package/.claude/helpers/gate.cjs +3 -3
- package/.claude/helpers/statusline.cjs +69 -33
- package/.claude/helpers/subagent-bootstrap.json +1 -1
- package/.claude/helpers/subagent-start.cjs +1 -1
- package/.claude/skills/eldar/SKILL.md +8 -0
- package/bin/build-embeddings.mjs +6 -20
- package/bin/cli.js +5 -0
- package/bin/gate.cjs +3 -3
- package/bin/generate-code-map.mjs +4 -24
- package/bin/hooks.mjs +3 -12
- package/bin/index-all.mjs +3 -13
- package/bin/index-guidance.mjs +59 -119
- package/bin/index-patterns.mjs +6 -24
- package/bin/index-tests.mjs +4 -23
- package/bin/lib/db-repair.mjs +4 -25
- package/bin/lib/get-backend.mjs +306 -0
- package/bin/lib/incremental-write.mjs +27 -7
- package/bin/lib/moflo-paths.mjs +64 -4
- package/bin/lib/suppress-sqlite-warning.mjs +57 -0
- package/bin/migrations/knowledge-purge.mjs +7 -8
- package/bin/migrations/knowledge-to-learnings.mjs +7 -9
- package/bin/migrations/purge-doc-entries.mjs +52 -0
- package/bin/migrations/strip-context-preambles.mjs +95 -0
- package/bin/run-migrations.mjs +1 -10
- package/bin/semantic-search.mjs +11 -19
- package/bin/session-start-launcher.mjs +102 -100
- package/bin/simplify-classify.cjs +38 -17
- package/dist/src/cli/commands/daemon.js +38 -11
- package/dist/src/cli/commands/doctor-checks-coverage-truth.js +136 -0
- package/dist/src/cli/commands/doctor-checks-memory-access.js +244 -5
- package/dist/src/cli/commands/doctor-checks-memory.js +13 -18
- package/dist/src/cli/commands/doctor-checks-version-skew.js +94 -0
- package/dist/src/cli/commands/doctor-checks-writers-audit.js +170 -0
- package/dist/src/cli/commands/doctor-embedding-hygiene.js +3 -15
- package/dist/src/cli/commands/doctor-fixes.js +30 -0
- package/dist/src/cli/commands/doctor-registry.js +14 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/embeddings.js +17 -22
- package/dist/src/cli/commands/memory.js +54 -75
- package/dist/src/cli/embeddings/persistent-cache.js +44 -83
- package/dist/src/cli/init/claudemd-generator.js +4 -0
- package/dist/src/cli/init/moflo-init.js +40 -0
- package/dist/src/cli/mcp-tools/memory-tools.js +177 -32
- package/dist/src/cli/memory/bridge-core.js +256 -30
- package/dist/src/cli/memory/bridge-entries.js +76 -8
- package/dist/src/cli/memory/controller-registry.js +7 -2
- package/dist/src/cli/memory/controllers/batch-operations.js +5 -1
- package/dist/src/cli/memory/controllers/hierarchical-memory.js +7 -2
- package/dist/src/cli/memory/controllers/mutation-guard.js +22 -2
- package/dist/src/cli/memory/daemon-backend.js +400 -0
- package/dist/src/cli/memory/daemon-write-client.js +192 -15
- package/dist/src/cli/memory/database-provider.js +57 -40
- package/dist/src/cli/memory/hnsw-persistence.js +6 -8
- package/dist/src/cli/memory/index.js +0 -1
- package/dist/src/cli/memory/memory-bridge.js +40 -8
- package/dist/src/cli/memory/memory-initializer.js +286 -220
- package/dist/src/cli/memory/rvf-migration.js +25 -11
- package/dist/src/cli/memory/sqlite-backend.js +573 -0
- package/dist/src/cli/memory/suppress-sqlite-warning.js +49 -0
- package/dist/src/cli/services/cherry-pick-learnings.js +32 -21
- package/dist/src/cli/services/daemon-dashboard.js +13 -1
- package/dist/src/cli/services/daemon-lock.js +58 -1
- package/dist/src/cli/services/daemon-memory-rpc.js +245 -10
- package/dist/src/cli/services/embeddings-migration.js +9 -12
- package/dist/src/cli/services/ephemeral-namespace-purge.js +6 -11
- package/dist/src/cli/services/learning-service.js +12 -20
- package/dist/src/cli/services/project-root.js +69 -9
- package/dist/src/cli/services/soft-delete-purge.js +6 -11
- package/dist/src/cli/services/sqljs-migration-store.js +4 -1
- package/dist/src/cli/services/subagent-bootstrap.js +1 -1
- package/dist/src/cli/shared/events/event-store.js +26 -55
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -4
- package/dist/src/cli/memory/sqljs-backend.js +0 -643
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writers Audit runtime doctor check (epic #1054.S5 / #1059).
|
|
3
|
+
*
|
|
4
|
+
* Runtime sibling of S1's static lint (`tests/system/moflo-db-writer-audit.test.ts`).
|
|
5
|
+
* Enumerates running node processes whose command line invokes one of the
|
|
6
|
+
* known cross-process writers (build-embeddings, migrations) and fails if any
|
|
7
|
+
* are alive while the daemon owns the lock.
|
|
8
|
+
*
|
|
9
|
+
* Why not lsof / handle.exe?
|
|
10
|
+
* - `lsof` not installed on every Linux distro and not on Windows.
|
|
11
|
+
* - `handle.exe` (Sysinternals) requires manual install.
|
|
12
|
+
* - `openfiles.exe` is disabled by default on Windows and requires a reboot to
|
|
13
|
+
* enable.
|
|
14
|
+
*
|
|
15
|
+
* Command-line signature scan is cross-platform, dependency-free, and matches
|
|
16
|
+
* the writers we actually care about — S3 ported them all to the daemon-offline
|
|
17
|
+
* pattern, so any of them being alive concurrently with the daemon is a
|
|
18
|
+
* regression of S3's wrapper logic.
|
|
19
|
+
*
|
|
20
|
+
* @module cli/commands/doctor-checks-writers-audit
|
|
21
|
+
*/
|
|
22
|
+
import { execSync } from 'child_process';
|
|
23
|
+
import { existsSync, readFileSync } from 'fs';
|
|
24
|
+
import { join } from 'path';
|
|
25
|
+
import { getDaemonLockHolder } from '../services/daemon-lock.js';
|
|
26
|
+
import { errorDetail } from '../shared/utils/error-detail.js';
|
|
27
|
+
const SCAN_TIMEOUT_MS_WIN = 10_000;
|
|
28
|
+
const SCAN_TIMEOUT_MS_POSIX = 5_000;
|
|
29
|
+
const CMDLINE_CAPTURE_LEN = 300;
|
|
30
|
+
/**
|
|
31
|
+
* Command-line fragments that identify a CROSS-PROCESS moflo.db writer.
|
|
32
|
+
* These should never run while the daemon owns the lock (S3 wraps every
|
|
33
|
+
* invocation in an explicit daemon-stop). Indexer scripts (index-guidance,
|
|
34
|
+
* index-tests, etc.) are daemon-spawned children — they appear in
|
|
35
|
+
* `background-pids.json` and are filtered out below.
|
|
36
|
+
*/
|
|
37
|
+
const FOREIGN_WRITER_PATTERNS = [
|
|
38
|
+
/build-embeddings\.mjs/i,
|
|
39
|
+
/bin[\\\/]migrations[\\\/][^\s"']+\.mjs/i,
|
|
40
|
+
/lib[\\\/]db-repair\.mjs/i,
|
|
41
|
+
];
|
|
42
|
+
function readTrackedBackgroundPids(cwd) {
|
|
43
|
+
const result = new Set();
|
|
44
|
+
const registryFile = join(cwd, '.moflo', 'background-pids.json');
|
|
45
|
+
try {
|
|
46
|
+
if (!existsSync(registryFile))
|
|
47
|
+
return result;
|
|
48
|
+
const entries = JSON.parse(readFileSync(registryFile, 'utf-8'));
|
|
49
|
+
if (!Array.isArray(entries))
|
|
50
|
+
return result;
|
|
51
|
+
for (const entry of entries) {
|
|
52
|
+
if (entry && typeof entry.pid === 'number' && entry.pid > 0) {
|
|
53
|
+
result.add(entry.pid);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
catch { /* unreadable — treat as empty */ }
|
|
58
|
+
return result;
|
|
59
|
+
}
|
|
60
|
+
function enumerateNodeProcesses() {
|
|
61
|
+
try {
|
|
62
|
+
if (process.platform === 'win32') {
|
|
63
|
+
const csv = execSync('powershell -NoProfile -Command "Get-CimInstance Win32_Process -Filter \\"Name=\'node.exe\'\\" | Select-Object ProcessId,CommandLine | ConvertTo-Csv -NoTypeInformation"', { encoding: 'utf-8', timeout: SCAN_TIMEOUT_MS_WIN, windowsHide: true });
|
|
64
|
+
const out = [];
|
|
65
|
+
for (const line of csv.split(/\r?\n/)) {
|
|
66
|
+
const m = line.match(/^"(\d+)","?(.*?)"?$/);
|
|
67
|
+
if (!m)
|
|
68
|
+
continue;
|
|
69
|
+
const pid = parseInt(m[1], 10);
|
|
70
|
+
if (!Number.isFinite(pid) || pid <= 0)
|
|
71
|
+
continue;
|
|
72
|
+
out.push({ pid, cmdline: m[2].replace(/""/g, '"').slice(0, CMDLINE_CAPTURE_LEN) });
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
// POSIX
|
|
77
|
+
const ps = execSync('ps -ww -eo pid,command', { encoding: 'utf-8', timeout: SCAN_TIMEOUT_MS_POSIX });
|
|
78
|
+
const out = [];
|
|
79
|
+
for (const line of ps.split(/\r?\n/)) {
|
|
80
|
+
const m = line.trim().match(/^(\d+)\s+(.*)$/);
|
|
81
|
+
if (!m)
|
|
82
|
+
continue;
|
|
83
|
+
const cmd = m[2];
|
|
84
|
+
if (!/\bnode\b/.test(cmd))
|
|
85
|
+
continue;
|
|
86
|
+
const pid = parseInt(m[1], 10);
|
|
87
|
+
if (!Number.isFinite(pid) || pid <= 0)
|
|
88
|
+
continue;
|
|
89
|
+
out.push({ pid, cmdline: cmd.slice(0, CMDLINE_CAPTURE_LEN) });
|
|
90
|
+
}
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return [];
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function formatCmdline(raw, max = 80) {
|
|
98
|
+
const cleaned = raw.replace(/^"?[^"\s]*node(?:\.exe)?"?\s+/i, '').trim();
|
|
99
|
+
return cleaned.length > max ? cleaned.slice(0, max - 1) + '…' : cleaned;
|
|
100
|
+
}
|
|
101
|
+
export function findForeignWriters(procs, daemonPid, trackedPids, selfPid) {
|
|
102
|
+
const out = [];
|
|
103
|
+
for (const p of procs) {
|
|
104
|
+
if (p.pid === selfPid)
|
|
105
|
+
continue;
|
|
106
|
+
if (p.pid === daemonPid)
|
|
107
|
+
continue;
|
|
108
|
+
if (trackedPids.has(p.pid))
|
|
109
|
+
continue;
|
|
110
|
+
for (const re of FOREIGN_WRITER_PATTERNS) {
|
|
111
|
+
const m = p.cmdline.match(re);
|
|
112
|
+
if (m) {
|
|
113
|
+
out.push({ ...p, matchedPattern: m[0] });
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Pass: daemon down (single-writer invariant has nothing to enforce) OR no
|
|
122
|
+
* foreign writers detected.
|
|
123
|
+
*
|
|
124
|
+
* Fail: daemon owns the lock AND a non-daemon, non-tracked node process is
|
|
125
|
+
* running a known cross-process writer. Lists every offender so the user can
|
|
126
|
+
* SIGKILL them by PID.
|
|
127
|
+
*/
|
|
128
|
+
export async function checkWritersAudit(cwd = process.cwd()) {
|
|
129
|
+
const name = 'Writers Audit';
|
|
130
|
+
try {
|
|
131
|
+
const daemonPid = getDaemonLockHolder(cwd);
|
|
132
|
+
if (daemonPid === null) {
|
|
133
|
+
return {
|
|
134
|
+
name,
|
|
135
|
+
status: 'pass',
|
|
136
|
+
message: 'Daemon not running — single-writer invariant trivially satisfied',
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
const procs = enumerateNodeProcesses();
|
|
140
|
+
const trackedPids = readTrackedBackgroundPids(cwd);
|
|
141
|
+
const foreign = findForeignWriters(procs, daemonPid, trackedPids, process.pid);
|
|
142
|
+
if (foreign.length === 0) {
|
|
143
|
+
return {
|
|
144
|
+
name,
|
|
145
|
+
status: 'pass',
|
|
146
|
+
message: `Daemon PID ${daemonPid} is sole writer; no foreign writers detected`,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const detail = foreign
|
|
150
|
+
.map((p) => `pid=${p.pid} (${p.matchedPattern}): ${formatCmdline(p.cmdline)}`)
|
|
151
|
+
.join(' | ');
|
|
152
|
+
return {
|
|
153
|
+
name,
|
|
154
|
+
status: 'fail',
|
|
155
|
+
message: `${foreign.length} non-daemon writer(s) running concurrently with daemon (PID ${daemonPid}): ${detail}. ` +
|
|
156
|
+
`Each will clobber the daemon's sql.js snapshot on flush — #1054 bug class.`,
|
|
157
|
+
fix: process.platform === 'win32'
|
|
158
|
+
? `taskkill /F /PID ${foreign.map((p) => p.pid).join(' /PID ')}`
|
|
159
|
+
: `kill ${foreign.map((p) => p.pid).join(' ')}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
return {
|
|
164
|
+
name,
|
|
165
|
+
status: 'warn',
|
|
166
|
+
message: `Unable to audit writers: ${errorDetail(e, { firstLineOnly: true })}`,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=doctor-checks-writers-audit.js.map
|
|
@@ -24,8 +24,8 @@
|
|
|
24
24
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
25
25
|
import { existsSync } from 'fs';
|
|
26
26
|
import { CANONICAL_EMBEDDING_MODEL } from '../embeddings/migration/types.js';
|
|
27
|
-
import { mofloImport } from '../services/moflo-require.js';
|
|
28
27
|
import { memoryDbCandidatePaths } from '../services/moflo-paths.js';
|
|
28
|
+
import { openDaemonDatabase } from '../memory/daemon-backend.js';
|
|
29
29
|
/**
|
|
30
30
|
* Known neural-model labels that all share the all-MiniLM-L6-v2 384-dim
|
|
31
31
|
* vector space. The Story-2 migration retags any of these to the
|
|
@@ -59,7 +59,7 @@ export async function checkEmbeddingHygiene() {
|
|
|
59
59
|
return {
|
|
60
60
|
name: 'Embedding hygiene',
|
|
61
61
|
status: 'pass',
|
|
62
|
-
message: 'Cannot inspect memory DB (
|
|
62
|
+
message: 'Cannot inspect memory DB (open failed)',
|
|
63
63
|
};
|
|
64
64
|
}
|
|
65
65
|
if (groups.length === 0) {
|
|
@@ -127,21 +127,9 @@ function resolveMemoryDb() {
|
|
|
127
127
|
return memoryDbCandidatePaths(process.cwd()).find((p) => existsSync(p)) ?? null;
|
|
128
128
|
}
|
|
129
129
|
async function loadModelGroups(dbPath) {
|
|
130
|
-
const fs = await import('fs');
|
|
131
|
-
let initSqlJs;
|
|
132
|
-
try {
|
|
133
|
-
initSqlJs = (await mofloImport('sql.js'))?.default;
|
|
134
|
-
}
|
|
135
|
-
catch {
|
|
136
|
-
return null;
|
|
137
|
-
}
|
|
138
|
-
if (!initSqlJs)
|
|
139
|
-
return null;
|
|
140
130
|
let db;
|
|
141
131
|
try {
|
|
142
|
-
|
|
143
|
-
const buffer = fs.readFileSync(dbPath);
|
|
144
|
-
db = new SQL.Database(buffer);
|
|
132
|
+
db = openDaemonDatabase(dbPath);
|
|
145
133
|
}
|
|
146
134
|
catch {
|
|
147
135
|
return null;
|
|
@@ -169,6 +169,36 @@ export async function autoFixCheck(check) {
|
|
|
169
169
|
catch { /* best effort */ }
|
|
170
170
|
return runFixCommand('npx moflo daemon start');
|
|
171
171
|
},
|
|
172
|
+
// Epic #1054.S5 / #1059 — SIGTERM the stale daemon and let the launcher's
|
|
173
|
+
// existing respawn path (mirrored as `npx moflo daemon start`) pick up the
|
|
174
|
+
// installed-version code. Mirrors `recycleDaemon` in
|
|
175
|
+
// bin/session-start-launcher.mjs so the auto-fix matches the launcher's
|
|
176
|
+
// behavior exactly.
|
|
177
|
+
'Daemon Version Skew': async () => {
|
|
178
|
+
const cwd = process.cwd();
|
|
179
|
+
const { getDaemonLockPayload } = await import('../services/daemon-lock.js');
|
|
180
|
+
const payload = getDaemonLockPayload(cwd);
|
|
181
|
+
if (payload?.pid && payload.pid > 0) {
|
|
182
|
+
try {
|
|
183
|
+
process.kill(payload.pid, 'SIGTERM');
|
|
184
|
+
}
|
|
185
|
+
catch { /* already dead */ }
|
|
186
|
+
}
|
|
187
|
+
const lockFile = join(cwd, '.moflo', 'daemon.lock');
|
|
188
|
+
try {
|
|
189
|
+
if (existsSync(lockFile))
|
|
190
|
+
unlinkSync(lockFile);
|
|
191
|
+
}
|
|
192
|
+
catch { /* ok */ }
|
|
193
|
+
return runFixCommand('npx moflo daemon start');
|
|
194
|
+
},
|
|
195
|
+
'Embedding Coverage Truth': async () => {
|
|
196
|
+
// Same as the existing Embeddings fix — rebuild the cache by re-running
|
|
197
|
+
// the embeddings pipeline. Routes through `npx moflo` so the consumer
|
|
198
|
+
// CLI resolution stays consistent across platforms (see
|
|
199
|
+
// feedback_cross_platform_mandatory).
|
|
200
|
+
return runFixCommand('npx moflo embeddings init --force');
|
|
201
|
+
},
|
|
172
202
|
'MCP Servers': async () => {
|
|
173
203
|
return runFixCommand('claude mcp add moflo -- npx -y moflo mcp start');
|
|
174
204
|
},
|
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { checkSubagentHealth, checkSpellExecution, checkMcpToolInvocation, checkHookExecution, checkMcpSpellIntegration, checkGateHealth, checkHookBlockDrift, checkMofloDbBridge, } from './doctor-checks-deep.js';
|
|
8
8
|
import { checkEmbeddingHygiene } from './doctor-embedding-hygiene.js';
|
|
9
|
+
import { checkDaemonVersionSkew } from './doctor-checks-version-skew.js';
|
|
10
|
+
import { checkEmbeddingCoverageTruth } from './doctor-checks-coverage-truth.js';
|
|
11
|
+
import { checkWritersAudit } from './doctor-checks-writers-audit.js';
|
|
9
12
|
import { checkSwarmFunctional, checkHiveMindFunctional, } from './doctor-checks-swarm.js';
|
|
10
13
|
import { checkMemoryAccessFunctional } from './doctor-checks-memory-access.js';
|
|
11
14
|
import { checkBuildTools, checkClaudeCode, checkDiskSpace, checkGit, checkGitRepo, checkNodeVersion, checkNpmVersion, } from './doctor-checks-runtime.js';
|
|
@@ -33,10 +36,13 @@ export const allChecks = [
|
|
|
33
36
|
checkMofloYamlCompliance,
|
|
34
37
|
checkStatusLine,
|
|
35
38
|
checkDaemonStatus,
|
|
39
|
+
checkDaemonVersionSkew,
|
|
36
40
|
checkDaemonWriteRouting,
|
|
41
|
+
checkWritersAudit,
|
|
37
42
|
checkMemoryDatabase,
|
|
38
43
|
checkEmbeddings,
|
|
39
44
|
checkEmbeddingHygiene,
|
|
45
|
+
checkEmbeddingCoverageTruth,
|
|
40
46
|
checkTestDirs,
|
|
41
47
|
checkMcpServers,
|
|
42
48
|
checkDiskSpace,
|
|
@@ -77,11 +83,19 @@ export const componentMap = {
|
|
|
77
83
|
'statusline': checkStatusLine,
|
|
78
84
|
'status-line': checkStatusLine,
|
|
79
85
|
'daemon': checkDaemonStatus,
|
|
86
|
+
'daemon-version-skew': checkDaemonVersionSkew,
|
|
87
|
+
'version-skew': checkDaemonVersionSkew,
|
|
88
|
+
'skew': checkDaemonVersionSkew,
|
|
80
89
|
'daemon-write-routing': checkDaemonWriteRouting,
|
|
81
90
|
'write-routing': checkDaemonWriteRouting,
|
|
91
|
+
'writers-audit': checkWritersAudit,
|
|
92
|
+
'writers': checkWritersAudit,
|
|
82
93
|
'memory': checkMemoryDatabase,
|
|
83
94
|
'embeddings': checkEmbeddings,
|
|
84
95
|
'embedding-hygiene': checkEmbeddingHygiene,
|
|
96
|
+
'embedding-coverage': checkEmbeddingCoverageTruth,
|
|
97
|
+
'coverage': checkEmbeddingCoverageTruth,
|
|
98
|
+
'coverage-truth': checkEmbeddingCoverageTruth,
|
|
85
99
|
'hygiene': checkEmbeddingHygiene,
|
|
86
100
|
'git': checkGit,
|
|
87
101
|
'mcp': checkMcpServers,
|
|
@@ -40,7 +40,7 @@ export const doctorCommand = {
|
|
|
40
40
|
{
|
|
41
41
|
name: 'component',
|
|
42
42
|
short: 'c',
|
|
43
|
-
description: 'Check specific component (version, node, npm, config, daemon, memory, embeddings, git, mcp, claude, disk, typescript, semantic, intelligence, swarm, hive-mind)',
|
|
43
|
+
description: 'Check specific component (version, version-skew, node, npm, config, daemon, writers-audit, memory, embeddings, coverage-truth, git, mcp, claude, disk, typescript, semantic, intelligence, swarm, hive-mind)',
|
|
44
44
|
type: 'string',
|
|
45
45
|
},
|
|
46
46
|
{
|
|
@@ -13,10 +13,10 @@
|
|
|
13
13
|
* Created with ❤️ by motailz.com
|
|
14
14
|
*/
|
|
15
15
|
import { output } from '../output.js';
|
|
16
|
-
import { mofloImport } from '../services/moflo-require.js';
|
|
17
16
|
import { runEmbeddingsMigrationIfNeeded } from '../services/embeddings-migration.js';
|
|
18
17
|
import { memoryDbPath, MEMORY_DB_FILE, MOFLO_DIR } from '../services/moflo-paths.js';
|
|
19
18
|
import * as embeddings from '../embeddings/index.js';
|
|
19
|
+
import { openDaemonDatabase } from '../memory/daemon-backend.js';
|
|
20
20
|
import { errorDetail } from '../shared/utils/error-detail.js';
|
|
21
21
|
const DEFAULT_DB_PATH_FLAG = `${MOFLO_DIR}/${MEMORY_DB_FILE}`;
|
|
22
22
|
// Generate subcommand - REAL implementation
|
|
@@ -90,7 +90,7 @@ const generateCommand = {
|
|
|
90
90
|
}
|
|
91
91
|
},
|
|
92
92
|
};
|
|
93
|
-
// Search subcommand - REAL implementation using
|
|
93
|
+
// Search subcommand - REAL implementation using node:sqlite via openDaemonDatabase
|
|
94
94
|
const searchCommand = {
|
|
95
95
|
name: 'search',
|
|
96
96
|
description: 'Semantic similarity search',
|
|
@@ -131,11 +131,8 @@ const searchCommand = {
|
|
|
131
131
|
output.printInfo('Run: claude-flow memory init');
|
|
132
132
|
return { success: false, exitCode: 1 };
|
|
133
133
|
}
|
|
134
|
-
//
|
|
135
|
-
const
|
|
136
|
-
const SQL = await initSqlJs();
|
|
137
|
-
const fileBuffer = fs.readFileSync(fullDbPath);
|
|
138
|
-
const db = new SQL.Database(fileBuffer);
|
|
134
|
+
// node:sqlite via the unified factory (Phase 5 / #1084).
|
|
135
|
+
const db = openDaemonDatabase(fullDbPath);
|
|
139
136
|
const startTime = Date.now();
|
|
140
137
|
// Generate embedding for query
|
|
141
138
|
const { generateEmbedding } = await import('../memory/memory-initializer.js');
|
|
@@ -343,7 +340,7 @@ const compareCommand = {
|
|
|
343
340
|
}
|
|
344
341
|
},
|
|
345
342
|
};
|
|
346
|
-
// Collections subcommand - REAL implementation using
|
|
343
|
+
// Collections subcommand - REAL implementation using node:sqlite via openDaemonDatabase
|
|
347
344
|
const collectionsCommand = {
|
|
348
345
|
name: 'collections',
|
|
349
346
|
description: 'Manage embedding collections (namespaces)',
|
|
@@ -374,11 +371,8 @@ const collectionsCommand = {
|
|
|
374
371
|
output.writeln(output.dim('No collections yet - initialize memory first'));
|
|
375
372
|
return { success: true, data: [] };
|
|
376
373
|
}
|
|
377
|
-
//
|
|
378
|
-
const
|
|
379
|
-
const SQL = await initSqlJs();
|
|
380
|
-
const fileBuffer = fs.readFileSync(fullDbPath);
|
|
381
|
-
const db = new SQL.Database(fileBuffer);
|
|
374
|
+
// node:sqlite via the unified factory (Phase 5 / #1084).
|
|
375
|
+
const db = openDaemonDatabase(fullDbPath);
|
|
382
376
|
// Get collection stats from database
|
|
383
377
|
const statsQuery = db.exec(`
|
|
384
378
|
SELECT
|
|
@@ -1159,17 +1153,18 @@ const cacheCommand = {
|
|
|
1159
1153
|
else {
|
|
1160
1154
|
sqliteSize = `${sizeBytes} B`;
|
|
1161
1155
|
}
|
|
1162
|
-
// Try to count real entries via
|
|
1156
|
+
// Try to count real entries via node:sqlite
|
|
1163
1157
|
try {
|
|
1164
|
-
const
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1158
|
+
const db = openDaemonDatabase(resolvedDbPath);
|
|
1159
|
+
try {
|
|
1160
|
+
const result = db.exec('SELECT COUNT(*) as count FROM embeddings');
|
|
1161
|
+
if (result.length > 0 && result[0].values.length > 0) {
|
|
1162
|
+
sqliteEntries = result[0].values[0][0];
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
finally {
|
|
1166
|
+
db.close();
|
|
1171
1167
|
}
|
|
1172
|
-
db.close();
|
|
1173
1168
|
}
|
|
1174
1169
|
catch {
|
|
1175
1170
|
// Estimate entries from file size (~1600 bytes per entry for 384-dim embeddings)
|
|
@@ -7,8 +7,7 @@ import * as pathModule from 'path';
|
|
|
7
7
|
import { output } from '../output.js';
|
|
8
8
|
import { select, confirm, input } from '../prompt.js';
|
|
9
9
|
import { callMCPTool, MCPClientError } from '../mcp-client.js';
|
|
10
|
-
import {
|
|
11
|
-
import { atomicWriteFileSync } from '../services/atomic-file-write.js';
|
|
10
|
+
import { openDaemonDatabase } from '../memory/daemon-backend.js';
|
|
12
11
|
import { errorDetail } from '../shared/utils/error-detail.js';
|
|
13
12
|
// Memory backends
|
|
14
13
|
const BACKENDS = [
|
|
@@ -242,7 +241,7 @@ const searchCommand = {
|
|
|
242
241
|
name: 'threshold',
|
|
243
242
|
description: 'Similarity threshold (0-1)',
|
|
244
243
|
type: 'number',
|
|
245
|
-
default: 0.
|
|
244
|
+
default: 0.5
|
|
246
245
|
},
|
|
247
246
|
{
|
|
248
247
|
name: 'type',
|
|
@@ -268,7 +267,8 @@ const searchCommand = {
|
|
|
268
267
|
const query = ctx.flags.query || ctx.args[0];
|
|
269
268
|
const namespace = ctx.flags.namespace || 'all';
|
|
270
269
|
const limit = ctx.flags.limit || 10;
|
|
271
|
-
|
|
270
|
+
// #1053 S6: align with MCP default — was 0.3 here vs 0.7 in option block.
|
|
271
|
+
const threshold = ctx.flags.threshold || 0.5;
|
|
272
272
|
const searchType = ctx.flags.type || 'semantic';
|
|
273
273
|
const buildHnsw = ctx.flags.buildHnsw;
|
|
274
274
|
if (!query) {
|
|
@@ -1264,23 +1264,10 @@ const initMemoryCommand = {
|
|
|
1264
1264
|
const DB_FILENAME = 'memory.db';
|
|
1265
1265
|
const SWARM_DIR = '.swarm';
|
|
1266
1266
|
async function openDb(cwd) {
|
|
1267
|
-
const fs = await import('fs');
|
|
1268
1267
|
const path = await import('path');
|
|
1269
|
-
const initSqlJs = (await mofloImport('sql.js')).default;
|
|
1270
|
-
const SQL = await initSqlJs();
|
|
1271
1268
|
const dbPath = path.join(cwd, SWARM_DIR, DB_FILENAME);
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
1275
|
-
}
|
|
1276
|
-
let db;
|
|
1277
|
-
if (fs.existsSync(dbPath)) {
|
|
1278
|
-
const buffer = fs.readFileSync(dbPath);
|
|
1279
|
-
db = new SQL.Database(buffer);
|
|
1280
|
-
}
|
|
1281
|
-
else {
|
|
1282
|
-
db = new SQL.Database();
|
|
1283
|
-
}
|
|
1269
|
+
// openDaemonDatabase ensures the parent directory exists and applies WAL.
|
|
1270
|
+
const db = openDaemonDatabase(dbPath);
|
|
1284
1271
|
// Ensure table exists
|
|
1285
1272
|
db.run(`
|
|
1286
1273
|
CREATE TABLE IF NOT EXISTS memory_entries (
|
|
@@ -1306,10 +1293,14 @@ async function openDb(cwd) {
|
|
|
1306
1293
|
`);
|
|
1307
1294
|
db.run(`CREATE INDEX IF NOT EXISTS idx_memory_key_ns ON memory_entries(key, namespace)`);
|
|
1308
1295
|
db.run(`CREATE INDEX IF NOT EXISTS idx_memory_namespace ON memory_entries(namespace)`);
|
|
1309
|
-
return { db, dbPath
|
|
1296
|
+
return { db, dbPath };
|
|
1310
1297
|
}
|
|
1311
|
-
|
|
1312
|
-
|
|
1298
|
+
/**
|
|
1299
|
+
* Close the DB handle. node:sqlite + WAL has already persisted every prior
|
|
1300
|
+
* `db.run` incrementally — the explicit atomicWriteFileSync sql.js used to
|
|
1301
|
+
* need is gone (Phase 5 / #1084).
|
|
1302
|
+
*/
|
|
1303
|
+
function saveAndCloseDb(db, _dbPath) {
|
|
1313
1304
|
db.close();
|
|
1314
1305
|
}
|
|
1315
1306
|
function batchGenerateId() {
|
|
@@ -1612,16 +1603,16 @@ const indexGuidanceCommand = {
|
|
|
1612
1603
|
try {
|
|
1613
1604
|
const content = fs.readFileSync(filePath, 'utf-8');
|
|
1614
1605
|
const contentHash_ = hashContent(content);
|
|
1615
|
-
//
|
|
1606
|
+
// #1053 S4: doc-* retired — read docContentHash off chunk-0 instead.
|
|
1616
1607
|
if (!forceReindex) {
|
|
1617
1608
|
const stmt = db.prepare('SELECT metadata FROM memory_entries WHERE key = ? AND namespace = ?');
|
|
1618
|
-
stmt.bind([
|
|
1609
|
+
stmt.bind([`${chunkPrefix}-0`, NAMESPACE]);
|
|
1619
1610
|
const entry = stmt.step() ? stmt.getAsObject() : null;
|
|
1620
1611
|
stmt.free();
|
|
1621
1612
|
if (entry?.metadata) {
|
|
1622
1613
|
try {
|
|
1623
1614
|
const meta = JSON.parse(entry.metadata);
|
|
1624
|
-
if (meta.
|
|
1615
|
+
if (meta.docContentHash === contentHash_) {
|
|
1625
1616
|
return { docKey, status: 'unchanged', chunks: 0 };
|
|
1626
1617
|
}
|
|
1627
1618
|
}
|
|
@@ -1630,19 +1621,12 @@ const indexGuidanceCommand = {
|
|
|
1630
1621
|
}
|
|
1631
1622
|
const stats = fs.statSync(filePath);
|
|
1632
1623
|
const relativePath = filePath.replace(cwd, '').replace(/\\/g, '/');
|
|
1633
|
-
// Delete old chunks
|
|
1624
|
+
// Delete old chunks. Also delete any legacy doc-* row (#1053 S4).
|
|
1634
1625
|
db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${chunkPrefix}%`]);
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
fileSize: stats.size,
|
|
1640
|
-
lastModified: stats.mtime.toISOString(),
|
|
1641
|
-
contentHash: contentHash_,
|
|
1642
|
-
indexedAt: new Date().toISOString(),
|
|
1643
|
-
ragVersion: '2.0',
|
|
1644
|
-
};
|
|
1645
|
-
batchStoreEntry(db, docKey, NAMESPACE, content, docMetadata, [keyPrefix, 'document']);
|
|
1626
|
+
db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, docKey]);
|
|
1627
|
+
// #1053 S4: doc-* entries no longer written. parentDoc on chunks
|
|
1628
|
+
// remains as an identifier label; callers Read parentPath when
|
|
1629
|
+
// they need the source file (see shipped/moflo-memory-protocol.md).
|
|
1646
1630
|
// Chunk content
|
|
1647
1631
|
const chunks = chunkMarkdown(content, fileName);
|
|
1648
1632
|
if (chunks.length === 0) {
|
|
@@ -1650,26 +1634,24 @@ const indexGuidanceCommand = {
|
|
|
1650
1634
|
}
|
|
1651
1635
|
const hierarchy = buildHierarchy(chunks, chunkPrefix);
|
|
1652
1636
|
const siblings = chunks.map((_, i) => `${chunkPrefix}-${i}`);
|
|
1653
|
-
// Update doc with children refs
|
|
1654
|
-
const docChildrenMeta = { ...docMetadata, children: siblings, chunkCount: chunks.length };
|
|
1655
|
-
batchStoreEntry(db, docKey, NAMESPACE, content, docChildrenMeta, [keyPrefix, 'document']);
|
|
1656
1637
|
for (let i = 0; i < chunks.length; i++) {
|
|
1657
1638
|
const chunk = chunks[i];
|
|
1658
1639
|
const chunkKey = `${chunkPrefix}-${i}`;
|
|
1659
1640
|
const prevChunk = i > 0 ? `${chunkPrefix}-${i - 1}` : null;
|
|
1660
1641
|
const nextChunk = i < chunks.length - 1 ? `${chunkPrefix}-${i + 1}` : null;
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
: null;
|
|
1664
|
-
const contextAfter = i < chunks.length - 1
|
|
1665
|
-
? extractOverlapContext(chunks[i + 1].content, overlapPercent, 'start')
|
|
1666
|
-
: null;
|
|
1642
|
+
// #1053 S5: dropped prev/next preamble wrapping. Traversal happens
|
|
1643
|
+
// via memory_get_neighbors now (S2).
|
|
1667
1644
|
const hierInfo = hierarchy[chunkKey];
|
|
1668
1645
|
const chunkMetadata = {
|
|
1669
1646
|
type: 'chunk',
|
|
1670
1647
|
ragVersion: '2.0',
|
|
1648
|
+
// #1053 S4: parentDoc is an identifier label (target row no
|
|
1649
|
+
// longer exists); use parentPath for the actual source file.
|
|
1650
|
+
// docContentHash on every chunk lets the skip-if-unchanged
|
|
1651
|
+
// check read it off chunk-0.
|
|
1671
1652
|
parentDoc: docKey,
|
|
1672
1653
|
parentPath: relativePath,
|
|
1654
|
+
docContentHash: contentHash_,
|
|
1673
1655
|
chunkIndex: i,
|
|
1674
1656
|
totalChunks: chunks.length,
|
|
1675
1657
|
prevChunk,
|
|
@@ -1682,21 +1664,12 @@ const indexGuidanceCommand = {
|
|
|
1682
1664
|
headerLine: chunk.headerLine,
|
|
1683
1665
|
isPart: chunk.isPart || false,
|
|
1684
1666
|
partNum: chunk.partNum || null,
|
|
1685
|
-
contextOverlapPercent: overlapPercent,
|
|
1686
|
-
hasContextBefore: !!contextBefore,
|
|
1687
|
-
hasContextAfter: !!contextAfter,
|
|
1688
1667
|
contentLength: chunk.content.length,
|
|
1689
1668
|
contentHash: hashContent(chunk.content),
|
|
1690
1669
|
indexedAt: new Date().toISOString(),
|
|
1691
1670
|
};
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
searchableContent += `[Context from previous section:]\n${contextBefore}\n\n---\n\n`;
|
|
1695
|
-
}
|
|
1696
|
-
searchableContent += chunk.content;
|
|
1697
|
-
if (contextAfter) {
|
|
1698
|
-
searchableContent += `\n\n---\n\n[Context from next section:]\n${contextAfter}`;
|
|
1699
|
-
}
|
|
1671
|
+
// #1053 S5: title heading + chunk body. No prev/next preamble.
|
|
1672
|
+
const searchableContent = `# ${chunk.title}\n\n${chunk.content}`;
|
|
1700
1673
|
batchStoreEntry(db, chunkKey, NAMESPACE, searchableContent, chunkMetadata, [keyPrefix, 'chunk', `level-${chunk.level}`, chunk.title.toLowerCase().replace(/[^a-z0-9]+/g, '-')]);
|
|
1701
1674
|
}
|
|
1702
1675
|
return { docKey, status: 'indexed', chunks: chunks.length };
|
|
@@ -1753,22 +1726,28 @@ const indexGuidanceCommand = {
|
|
|
1753
1726
|
errors++;
|
|
1754
1727
|
}
|
|
1755
1728
|
}
|
|
1756
|
-
// Clean stale
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1729
|
+
// #1053 S4: Clean stale chunks for deleted files.
|
|
1730
|
+
// doc-* markers are gone — derive prefixes from chunk keys directly.
|
|
1731
|
+
// Chunk key shape: chunk-guidance-<filename>-<index>; group by stripping
|
|
1732
|
+
// the trailing -<index>.
|
|
1733
|
+
const chunksStmt = db.prepare(`SELECT DISTINCT key FROM memory_entries WHERE namespace = ? AND key LIKE 'chunk-guidance-%'`);
|
|
1734
|
+
chunksStmt.bind([NAMESPACE]);
|
|
1735
|
+
const seenPrefixes = new Set();
|
|
1736
|
+
while (chunksStmt.step()) {
|
|
1737
|
+
const { key } = chunksStmt.getAsObject();
|
|
1738
|
+
const prefix = key.replace(/-\d+$/, '');
|
|
1739
|
+
seenPrefixes.add(prefix);
|
|
1740
|
+
}
|
|
1741
|
+
chunksStmt.free();
|
|
1742
|
+
for (const prefix of seenPrefixes) {
|
|
1743
|
+
const filename = prefix.replace('chunk-guidance-', '') + '.md';
|
|
1744
|
+
const checkPath = pathModule.resolve(cwd, '.claude/guidance', filename);
|
|
1767
1745
|
if (!fs.existsSync(checkPath)) {
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1746
|
+
db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${prefix}-%`]);
|
|
1747
|
+
// Also sweep any legacy doc-* row for this prefix (one-time tidy).
|
|
1748
|
+
const legacyDocKey = prefix.replace('chunk-', 'doc-');
|
|
1749
|
+
db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, legacyDocKey]);
|
|
1750
|
+
output.writeln(output.dim(` Removed stale: ${prefix}-* (file ${filename} not found)`));
|
|
1772
1751
|
}
|
|
1773
1752
|
}
|
|
1774
1753
|
}
|
|
@@ -1948,9 +1927,9 @@ const rebuildIndexCommand = {
|
|
|
1948
1927
|
}
|
|
1949
1928
|
failed++;
|
|
1950
1929
|
}
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1930
|
+
// node:sqlite + WAL persists each db.run incrementally — the
|
|
1931
|
+
// periodic batch flush sql.js needed here was the export-+-rewrite
|
|
1932
|
+
// pattern Phase 5 (#1084) killed. No flush needed.
|
|
1954
1933
|
}
|
|
1955
1934
|
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
1956
1935
|
// Final stats
|