moflo 4.9.37 → 4.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/.claude/guidance/shipped/moflo-memory-protocol.md +5 -1
  2. package/.claude/guidance/shipped/moflo-memorydb-maintenance.md +22 -11
  3. package/.claude/guidance/shipped/moflo-root-cause-discipline.md +47 -0
  4. package/.claude/helpers/statusline.cjs +69 -33
  5. package/.claude/helpers/subagent-bootstrap.json +1 -1
  6. package/.claude/helpers/subagent-start.cjs +1 -1
  7. package/bin/build-embeddings.mjs +6 -20
  8. package/bin/cli.js +5 -0
  9. package/bin/generate-code-map.mjs +4 -24
  10. package/bin/hooks.mjs +3 -12
  11. package/bin/index-all.mjs +3 -13
  12. package/bin/index-guidance.mjs +36 -85
  13. package/bin/index-patterns.mjs +6 -24
  14. package/bin/index-tests.mjs +4 -23
  15. package/bin/lib/db-repair.mjs +4 -25
  16. package/bin/lib/get-backend.mjs +306 -0
  17. package/bin/lib/incremental-write.mjs +27 -7
  18. package/bin/lib/moflo-paths.mjs +64 -4
  19. package/bin/lib/suppress-sqlite-warning.mjs +57 -0
  20. package/bin/migrations/knowledge-purge.mjs +7 -8
  21. package/bin/migrations/knowledge-to-learnings.mjs +7 -9
  22. package/bin/migrations/purge-doc-entries.mjs +7 -8
  23. package/bin/migrations/strip-context-preambles.mjs +4 -6
  24. package/bin/run-migrations.mjs +1 -10
  25. package/bin/semantic-search.mjs +7 -18
  26. package/bin/session-start-launcher.mjs +102 -102
  27. package/bin/simplify-classify.cjs +38 -17
  28. package/dist/src/cli/commands/daemon.js +38 -11
  29. package/dist/src/cli/commands/doctor-checks-coverage-truth.js +136 -0
  30. package/dist/src/cli/commands/doctor-checks-memory-access.js +146 -86
  31. package/dist/src/cli/commands/doctor-checks-memory.js +13 -18
  32. package/dist/src/cli/commands/doctor-checks-version-skew.js +94 -0
  33. package/dist/src/cli/commands/doctor-checks-writers-audit.js +170 -0
  34. package/dist/src/cli/commands/doctor-embedding-hygiene.js +3 -15
  35. package/dist/src/cli/commands/doctor-fixes.js +30 -0
  36. package/dist/src/cli/commands/doctor-registry.js +14 -0
  37. package/dist/src/cli/commands/doctor.js +1 -1
  38. package/dist/src/cli/commands/embeddings.js +17 -22
  39. package/dist/src/cli/commands/memory.js +13 -23
  40. package/dist/src/cli/embeddings/persistent-cache.js +44 -83
  41. package/dist/src/cli/init/moflo-init.js +40 -0
  42. package/dist/src/cli/mcp-tools/memory-tools.js +10 -3
  43. package/dist/src/cli/memory/bridge-core.js +256 -30
  44. package/dist/src/cli/memory/bridge-entries.js +70 -6
  45. package/dist/src/cli/memory/controller-registry.js +7 -2
  46. package/dist/src/cli/memory/controllers/batch-operations.js +5 -1
  47. package/dist/src/cli/memory/controllers/hierarchical-memory.js +7 -2
  48. package/dist/src/cli/memory/controllers/mutation-guard.js +22 -2
  49. package/dist/src/cli/memory/daemon-backend.js +400 -0
  50. package/dist/src/cli/memory/daemon-write-client.js +192 -15
  51. package/dist/src/cli/memory/database-provider.js +57 -40
  52. package/dist/src/cli/memory/hnsw-persistence.js +6 -8
  53. package/dist/src/cli/memory/index.js +0 -1
  54. package/dist/src/cli/memory/memory-bridge.js +40 -8
  55. package/dist/src/cli/memory/memory-initializer.js +269 -209
  56. package/dist/src/cli/memory/rvf-migration.js +25 -11
  57. package/dist/src/cli/memory/sqlite-backend.js +573 -0
  58. package/dist/src/cli/memory/suppress-sqlite-warning.js +49 -0
  59. package/dist/src/cli/services/cherry-pick-learnings.js +32 -21
  60. package/dist/src/cli/services/daemon-dashboard.js +13 -1
  61. package/dist/src/cli/services/daemon-lock.js +58 -1
  62. package/dist/src/cli/services/daemon-memory-rpc.js +245 -10
  63. package/dist/src/cli/services/embeddings-migration.js +9 -12
  64. package/dist/src/cli/services/ephemeral-namespace-purge.js +6 -11
  65. package/dist/src/cli/services/learning-service.js +12 -20
  66. package/dist/src/cli/services/project-root.js +69 -9
  67. package/dist/src/cli/services/soft-delete-purge.js +6 -11
  68. package/dist/src/cli/services/sqljs-migration-store.js +4 -1
  69. package/dist/src/cli/services/subagent-bootstrap.js +1 -1
  70. package/dist/src/cli/shared/events/event-store.js +26 -55
  71. package/dist/src/cli/version.js +1 -1
  72. package/package.json +2 -4
  73. package/dist/src/cli/memory/sqljs-backend.js +0 -643
@@ -1,26 +1,33 @@
1
1
  /**
2
2
  * V3 Memory Initializer
3
- * Properly initializes the memory database with sql.js (WASM SQLite)
4
- * Includes pattern tables, vector embeddings, migration state tracking
3
+ * Properly initializes the memory database with node:sqlite via the
4
+ * unified `openDaemonDatabase` factory.
5
+ * Includes pattern tables, vector embeddings, migration state tracking.
5
6
  *
6
7
  * ADR-053: Routes through ControllerRegistry → AgentDB v3 when available,
7
- * falls back to raw sql.js for backwards compatibility.
8
+ * falls back to a direct node:sqlite write for backwards compatibility.
9
+ *
10
+ * Phase 5 (#1084): every `mofloImport('sql.js') + new SQL.Database(buffer)`
11
+ * triplet (and the matching `atomicWriteFileSync(path, db.export())`) was
12
+ * replaced with `openDaemonDatabase(path)`. node:sqlite + WAL persists each
13
+ * mutation incrementally; the explicit whole-file-dump that sql.js needed
14
+ * was the multi-writer clobber vector epic #1078 killed structurally.
8
15
  *
9
16
  * @module v3/cli/memory-initializer
10
17
  */
11
18
  import * as fs from 'fs';
12
19
  import * as path from 'path';
13
- import { mofloImport } from '../services/moflo-require.js';
14
- import { atomicWriteFileSync } from '../services/atomic-file-write.js';
15
20
  import { formatEmbeddingError } from './embedding-errors.js';
16
21
  import { HnswLite } from './hnsw-lite.js';
17
22
  import { tryLoadHnswSidecar } from './hnsw-persistence.js';
18
23
  import { EMBEDDING_MODEL_OPT_OUT, EPHEMERAL_NAMESPACES, getBridgeEmbedder } from './bridge-embedder.js';
19
24
  import { parseEmbeddingJson, toFloat32 } from './controllers/_shared.js';
20
25
  import { writeVectorStatsJson } from './bridge-core.js';
26
+ import { serialiseMetadata } from './bridge-entries.js';
21
27
  import { errorDetail } from '../shared/utils/error-detail.js';
22
28
  import { MOFLO_DIR, hnswIndexPath, legacyMemoryDbPath, memoryDbPath, } from '../services/moflo-paths.js';
23
- import { tryDaemonStore, tryDaemonDelete } from './daemon-write-client.js';
29
+ import { tryDaemonStore, tryDaemonDelete, tryDaemonGet, tryDaemonSearch, tryDaemonList } from './daemon-write-client.js';
30
+ import { openDaemonDatabase } from './daemon-backend.js';
24
31
  // #981 — daemon-write-client throws are a contract violation (it's documented
25
32
  // as never-throw). When a throw escapes anyway, log to stderr ONCE per process
26
33
  // and fall through to the direct-write path. Silent swallow would hide bugs;
@@ -34,8 +41,8 @@ function logRoutingFault(err) {
34
41
  }
35
42
  /**
36
43
  * Write vector-stats.json cache for the statusline (no subprocess needed).
37
- * Called after memory store in the raw-sql.js fallback path. The bridge path
38
- * goes through refreshVectorStatsCache() in bridge-core.ts instead.
44
+ * Called after memory store in the direct-write fallback path. The bridge
45
+ * path goes through refreshVectorStatsCache() in bridge-core.ts instead.
39
46
  * @param dbPath - path to the SQLite database file
40
47
  * @param stats - exact counts from a db query already in progress (required —
41
48
  * making this optional caused issue #639 by silently writing 0)
@@ -428,10 +435,7 @@ export async function getHNSWIndex(options) {
428
435
  const SELECT_METADATA_ONLY = `id, key, namespace, content, metadata`;
429
436
  if (fs.existsSync(dbPath)) {
430
437
  try {
431
- const initSqlJs = (await mofloImport('sql.js')).default;
432
- const SQL = await initSqlJs();
433
- const fileBuffer = fs.readFileSync(dbPath);
434
- const sqlDb = new SQL.Database(fileBuffer);
438
+ const sqlDb = openDaemonDatabase(dbPath);
435
439
  const cols = sidecarLoaded ? SELECT_METADATA_ONLY : SELECT_WITH_EMBEDDING;
436
440
  const result = sqlDb.exec(`
437
441
  SELECT ${cols}
@@ -834,10 +838,7 @@ export async function ensureSchemaColumns(dbPath) {
834
838
  if (!fs.existsSync(dbPath)) {
835
839
  return { success: true, columnsAdded: [] };
836
840
  }
837
- const initSqlJs = (await mofloImport('sql.js')).default;
838
- const SQL = await initSqlJs();
839
- const fileBuffer = fs.readFileSync(dbPath);
840
- const db = new SQL.Database(fileBuffer);
841
+ const db = openDaemonDatabase(dbPath);
841
842
  // Get current columns in memory_entries
842
843
  const tableInfo = db.exec("PRAGMA table_info(memory_entries)");
843
844
  const existingColumns = new Set(tableInfo[0]?.values?.map(row => row[1]) || []);
@@ -857,22 +858,17 @@ export async function ensureSchemaColumns(dbPath) {
857
858
  { name: 'access_count', definition: 'access_count INTEGER DEFAULT 0' },
858
859
  { name: 'status', definition: "status TEXT DEFAULT 'active'" }
859
860
  ];
860
- let modified = false;
861
861
  for (const col of requiredColumns) {
862
862
  if (!existingColumns.has(col.name)) {
863
863
  try {
864
864
  db.run(`ALTER TABLE memory_entries ADD COLUMN ${col.definition}`);
865
865
  columnsAdded.push(col.name);
866
- modified = true;
867
866
  }
868
867
  catch (e) {
869
868
  // Column might already exist or other error - continue
870
869
  }
871
870
  }
872
871
  }
873
- if (modified) {
874
- atomicWriteFileSync(dbPath, db.export());
875
- }
876
872
  db.close();
877
873
  return { success: true, columnsAdded };
878
874
  }
@@ -904,10 +900,7 @@ export async function checkAndMigrateLegacy(options) {
904
900
  for (const legacyPath of legacyPaths) {
905
901
  if (fs.existsSync(legacyPath) && legacyPath !== dbPath) {
906
902
  try {
907
- const initSqlJs = (await mofloImport('sql.js')).default;
908
- const SQL = await initSqlJs();
909
- const legacyBuffer = fs.readFileSync(legacyPath);
910
- const legacyDb = new SQL.Database(legacyBuffer);
903
+ const legacyDb = openDaemonDatabase(legacyPath);
911
904
  // Check if it has data
912
905
  const countResult = legacyDb.exec('SELECT COUNT(*) FROM memory_entries');
913
906
  const count = countResult[0]?.values[0]?.[0] || 0;
@@ -980,7 +973,43 @@ async function activateControllerRegistry(dbPath, verbose) {
980
973
  return { activated, failed, initTimeMs: performance.now() - startTime };
981
974
  }
982
975
  /**
983
- * Initialize the memory database properly using sql.js
976
+ * Cross-platform safe unlink with a short EBUSY/EPERM retry loop. Windows
977
+ * holds file handles briefly after process exit (and the consumer's
978
+ * background daemon may still own the moflo.db handle when `memory init
979
+ * --force` runs); on POSIX the first attempt always succeeds. Total wait
980
+ * is bounded at ~750ms so we never paper over a real long-held handle.
981
+ *
982
+ * Used by `initializeMemoryDatabase` to ride out the daemon's brief handle
983
+ * release race after `flo healer --kill-zombies`. See #1098 for the smoke
984
+ * harness incident that drove this — the daemon's WAL+SHM file handles
985
+ * weren't released by Windows for ~200ms after SIGKILL.
986
+ */
987
+ function unlinkWithRetry(filePath) {
988
+ const delays = [0, 50, 100, 100, 100, 100, 100, 100, 50, 50]; // ~750ms total
989
+ let lastErr = null;
990
+ for (let attempt = 0; attempt < delays.length; attempt++) {
991
+ if (delays[attempt] > 0)
992
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delays[attempt]);
993
+ try {
994
+ fs.unlinkSync(filePath);
995
+ return;
996
+ }
997
+ catch (err) {
998
+ lastErr = err;
999
+ const code = err?.code;
1000
+ // Only retry on the handle-release race codes. ENOENT means we've
1001
+ // already won (or the file was never there); EACCES could be a real
1002
+ // permission issue worth surfacing immediately.
1003
+ if (code === 'ENOENT')
1004
+ return;
1005
+ if (code !== 'EBUSY' && code !== 'EPERM')
1006
+ throw err;
1007
+ }
1008
+ }
1009
+ throw lastErr;
1010
+ }
1011
+ /**
1012
+ * Initialize the memory database via the unified node:sqlite factory.
984
1013
  */
985
1014
  export async function initializeMemoryDatabase(options) {
986
1015
  const { backend = 'hybrid', dbPath: customPath, force = false, verbose = false, migrate = true } = options;
@@ -1017,130 +1046,91 @@ export async function initializeMemoryDatabase(options) {
1017
1046
  error: 'Database already exists. Use --force to reinitialize.'
1018
1047
  };
1019
1048
  }
1020
- // Try to use sql.js (WASM SQLite)
1021
- let db;
1022
- let usedSqlJs = false;
1023
- try {
1024
- // Dynamic import of sql.js
1025
- const initSqlJs = (await mofloImport('sql.js')).default;
1026
- const SQL = await initSqlJs();
1027
- // Load existing database or create new
1028
- if (fs.existsSync(dbPath) && force) {
1029
- fs.unlinkSync(dbPath);
1049
+ // Force a clean slate so the new file gets fresh WAL state too.
1050
+ if (fs.existsSync(dbPath) && force) {
1051
+ // Windows EBUSY guard (#1098): the consumer's background daemon
1052
+ // (spawned by session-start during `npm install`) holds an open
1053
+ // file handle on moflo.db. `unlinkSync` would otherwise throw EBUSY
1054
+ // immediately and the OS-level handle release race only resolves
1055
+ // after the daemon process actually exits, so a tight retry loop
1056
+ // can't outwait it. Stop the daemon first (graceful SIGTERM +
1057
+ // 1s grace + SIGKILL via `killBackgroundDaemon`), THEN retry the
1058
+ // unlink to ride out any residual handle-release lag.
1059
+ const projectRoot = path.dirname(path.dirname(dbPath));
1060
+ const { killBackgroundDaemon } = await import('../commands/daemon.js');
1061
+ try {
1062
+ await killBackgroundDaemon(projectRoot);
1030
1063
  }
1031
- db = new SQL.Database();
1032
- usedSqlJs = true;
1033
- }
1034
- catch (e) {
1035
- // sql.js not available, fall back to writing schema file
1036
- if (verbose) {
1037
- console.log('sql.js not available, writing schema file for later initialization');
1064
+ catch { /* best-effort; the retry below still gives us a budget */ }
1065
+ unlinkWithRetry(dbPath);
1066
+ // Also drop any sidecar WAL files so the next open doesn't replay
1067
+ // stale uncommitted transactions from the previous DB.
1068
+ for (const suffix of ['-wal', '-shm']) {
1069
+ const sidecar = dbPath + suffix;
1070
+ if (fs.existsSync(sidecar)) {
1071
+ try {
1072
+ unlinkWithRetry(sidecar);
1073
+ }
1074
+ catch { /* best-effort */ }
1075
+ }
1038
1076
  }
1039
1077
  }
1040
- if (usedSqlJs && db) {
1078
+ const db = openDaemonDatabase(dbPath);
1079
+ try {
1041
1080
  // Execute schema
1042
1081
  db.run(MEMORY_SCHEMA_V3);
1043
1082
  // Insert initial metadata
1044
1083
  db.run(getInitialMetadata(backend));
1045
- // Save to file
1046
- const data = db.export();
1047
- const buffer = Buffer.from(data);
1048
- fs.writeFileSync(dbPath, buffer);
1049
- // Close database
1050
- db.close();
1051
- // Also create schema file for reference
1052
- const schemaPath = path.join(dbDir, 'schema.sql');
1053
- fs.writeFileSync(schemaPath, MEMORY_SCHEMA_V3 + '\n' + getInitialMetadata(backend));
1054
- // ADR-053: Activate ControllerRegistry so controllers (ReasoningBank,
1055
- // SkillLibrary, ExplainableRecall, etc.) are instantiated during init
1056
- const controllerResult = await activateControllerRegistry(dbPath, verbose);
1057
- return {
1058
- success: true,
1059
- backend,
1060
- dbPath,
1061
- schemaVersion: '3.0.0',
1062
- tablesCreated: [
1063
- 'memory_entries',
1064
- 'patterns',
1065
- 'pattern_history',
1066
- 'trajectories',
1067
- 'trajectory_steps',
1068
- 'migration_state',
1069
- 'sessions',
1070
- 'vector_indexes',
1071
- 'metadata'
1072
- ],
1073
- indexesCreated: [
1074
- 'idx_memory_namespace',
1075
- 'idx_memory_key',
1076
- 'idx_memory_type',
1077
- 'idx_memory_status',
1078
- 'idx_memory_created',
1079
- 'idx_memory_accessed',
1080
- 'idx_memory_owner',
1081
- 'idx_patterns_type',
1082
- 'idx_patterns_confidence',
1083
- 'idx_patterns_status',
1084
- 'idx_patterns_last_matched',
1085
- 'idx_pattern_history_pattern',
1086
- 'idx_steps_trajectory'
1087
- ],
1088
- features: {
1089
- vectorEmbeddings: true,
1090
- patternLearning: true,
1091
- temporalDecay: true,
1092
- hnswIndexing: true,
1093
- migrationTracking: true
1094
- },
1095
- controllers: controllerResult,
1096
- };
1097
1084
  }
1098
- else {
1099
- // Fall back to schema file approach
1100
- const schemaPath = path.join(dbDir, 'schema.sql');
1101
- fs.writeFileSync(schemaPath, MEMORY_SCHEMA_V3 + '\n' + getInitialMetadata(backend));
1102
- // Create minimal valid SQLite file
1103
- const sqliteHeader = Buffer.alloc(4096, 0);
1104
- // SQLite format 3 header
1105
- Buffer.from('SQLite format 3\0').copy(sqliteHeader, 0);
1106
- sqliteHeader[16] = 0x10; // page size high byte (4096)
1107
- sqliteHeader[17] = 0x00; // page size low byte
1108
- sqliteHeader[18] = 0x01; // file format write version
1109
- sqliteHeader[19] = 0x01; // file format read version
1110
- sqliteHeader[24] = 0x00; // max embedded payload
1111
- sqliteHeader[25] = 0x40;
1112
- sqliteHeader[26] = 0x20; // min embedded payload
1113
- sqliteHeader[27] = 0x20; // leaf payload
1114
- fs.writeFileSync(dbPath, sqliteHeader);
1115
- // ADR-053: Activate ControllerRegistry even on fallback path
1116
- const controllerResult = await activateControllerRegistry(dbPath, verbose);
1117
- return {
1118
- success: true,
1119
- backend,
1120
- dbPath,
1121
- schemaVersion: '3.0.0',
1122
- tablesCreated: [
1123
- 'memory_entries (pending)',
1124
- 'patterns (pending)',
1125
- 'pattern_history (pending)',
1126
- 'trajectories (pending)',
1127
- 'trajectory_steps (pending)',
1128
- 'migration_state (pending)',
1129
- 'sessions (pending)',
1130
- 'vector_indexes (pending)',
1131
- 'metadata (pending)'
1132
- ],
1133
- indexesCreated: [],
1134
- features: {
1135
- vectorEmbeddings: true,
1136
- patternLearning: true,
1137
- temporalDecay: true,
1138
- hnswIndexing: true,
1139
- migrationTracking: true
1140
- },
1141
- controllers: controllerResult,
1142
- };
1085
+ finally {
1086
+ db.close();
1143
1087
  }
1088
+ // Also create schema file for reference
1089
+ const schemaPath = path.join(dbDir, 'schema.sql');
1090
+ fs.writeFileSync(schemaPath, MEMORY_SCHEMA_V3 + '\n' + getInitialMetadata(backend));
1091
+ // ADR-053: Activate ControllerRegistry so controllers (ReasoningBank,
1092
+ // SkillLibrary, ExplainableRecall, etc.) are instantiated during init
1093
+ const controllerResult = await activateControllerRegistry(dbPath, verbose);
1094
+ return {
1095
+ success: true,
1096
+ backend,
1097
+ dbPath,
1098
+ schemaVersion: '3.0.0',
1099
+ tablesCreated: [
1100
+ 'memory_entries',
1101
+ 'patterns',
1102
+ 'pattern_history',
1103
+ 'trajectories',
1104
+ 'trajectory_steps',
1105
+ 'migration_state',
1106
+ 'sessions',
1107
+ 'vector_indexes',
1108
+ 'metadata'
1109
+ ],
1110
+ indexesCreated: [
1111
+ 'idx_memory_namespace',
1112
+ 'idx_memory_key',
1113
+ 'idx_memory_type',
1114
+ 'idx_memory_status',
1115
+ 'idx_memory_created',
1116
+ 'idx_memory_accessed',
1117
+ 'idx_memory_owner',
1118
+ 'idx_patterns_type',
1119
+ 'idx_patterns_confidence',
1120
+ 'idx_patterns_status',
1121
+ 'idx_patterns_last_matched',
1122
+ 'idx_pattern_history_pattern',
1123
+ 'idx_steps_trajectory'
1124
+ ],
1125
+ features: {
1126
+ vectorEmbeddings: true,
1127
+ patternLearning: true,
1128
+ temporalDecay: true,
1129
+ hnswIndexing: true,
1130
+ migrationTracking: true
1131
+ },
1132
+ controllers: controllerResult,
1133
+ };
1144
1134
  }
1145
1135
  catch (error) {
1146
1136
  return {
@@ -1170,11 +1160,7 @@ export async function checkMemoryInitialization(dbPath) {
1170
1160
  return { initialized: false };
1171
1161
  }
1172
1162
  try {
1173
- // Try to load with sql.js
1174
- const initSqlJs = (await mofloImport('sql.js')).default;
1175
- const SQL = await initSqlJs();
1176
- const fileBuffer = fs.readFileSync(path_);
1177
- const db = new SQL.Database(fileBuffer);
1163
+ const db = openDaemonDatabase(path_);
1178
1164
  // Check for metadata table
1179
1165
  const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
1180
1166
  const tableNames = tables[0]?.values?.map(v => v[0]) || [];
@@ -1215,10 +1201,7 @@ export async function checkMemoryInitialization(dbPath) {
1215
1201
  export async function applyTemporalDecay(dbPath) {
1216
1202
  const path_ = dbPath || memoryDbPath(process.cwd());
1217
1203
  try {
1218
- const initSqlJs = (await mofloImport('sql.js')).default;
1219
- const SQL = await initSqlJs();
1220
- const fileBuffer = fs.readFileSync(path_);
1221
- const db = new SQL.Database(fileBuffer);
1204
+ const db = openDaemonDatabase(path_);
1222
1205
  // Apply decay: confidence *= exp(-decay_rate * days_since_last_use)
1223
1206
  const now = Date.now();
1224
1207
  const decayQuery = `
@@ -1232,7 +1215,6 @@ export async function applyTemporalDecay(dbPath) {
1232
1215
  `;
1233
1216
  db.run(decayQuery, [now, now, now]);
1234
1217
  const changes = db.getRowsModified();
1235
- atomicWriteFileSync(path_, db.export());
1236
1218
  db.close();
1237
1219
  return {
1238
1220
  success: true,
@@ -1399,12 +1381,7 @@ export async function verifyMemoryInit(dbPath, options) {
1399
1381
  const { verbose = false } = options || {};
1400
1382
  const tests = [];
1401
1383
  try {
1402
- const initSqlJs = (await mofloImport('sql.js')).default;
1403
- const SQL = await initSqlJs();
1404
- const fs = await import('fs');
1405
- // Load database
1406
- const fileBuffer = fs.readFileSync(dbPath);
1407
- const db = new SQL.Database(fileBuffer);
1384
+ const db = openDaemonDatabase(dbPath);
1408
1385
  // Test 1: Schema verification
1409
1386
  const schemaStart = Date.now();
1410
1387
  const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
@@ -1532,9 +1509,10 @@ export async function verifyMemoryInit(dbPath, options) {
1532
1509
  duration: Date.now() - indexStart
1533
1510
  });
1534
1511
  }
1535
- // Cleanup test entry
1512
+ // Cleanup test entry — WAL persists this DELETE incrementally; no
1513
+ // export+rewrite needed (the sql.js whole-file dump that lived here
1514
+ // was the multi-writer clobber vector epic #1078 killed).
1536
1515
  db.run(`DELETE FROM memory_entries WHERE id = ?`, [testId]);
1537
- atomicWriteFileSync(dbPath, db.export());
1538
1516
  db.close();
1539
1517
  const passed = tests.filter(t => t.passed).length;
1540
1518
  const failed = tests.filter(t => !t.passed).length;
@@ -1561,8 +1539,8 @@ export async function verifyMemoryInit(dbPath, options) {
1561
1539
  }
1562
1540
  }
1563
1541
  /**
1564
- * Store an entry directly using sql.js
1565
- * This bypasses MCP and writes directly to the database
1542
+ * Store an entry directly via node:sqlite.
1543
+ * This bypasses MCP and writes directly to the database.
1566
1544
  */
1567
1545
  export async function storeEntry(options) {
1568
1546
  // Soft-redirect: `knowledge` is a deprecated alias for `learnings`. Writes
@@ -1579,9 +1557,9 @@ export async function storeEntry(options) {
1579
1557
  }
1580
1558
  // #981 — single-writer routing. When an external daemon is reachable AND
1581
1559
  // we're not the daemon ourselves AND no custom dbPath was supplied, route
1582
- // the write through the daemon's HTTP RPC so its in-memory sql.js handle
1583
- // stays authoritative. Any failure path falls through to the existing
1584
- // bridge / raw-sql.js logic below — byte-identical behaviour to today.
1560
+ // the write through the daemon's HTTP RPC so its in-memory handle stays
1561
+ // authoritative. Any failure path falls through to the existing bridge /
1562
+ // direct-write logic below — byte-identical behaviour to today.
1585
1563
  if (!options.dbPath
1586
1564
  && process.env.MOFLO_IS_DAEMON !== '1'
1587
1565
  && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
@@ -1592,9 +1570,19 @@ export async function storeEntry(options) {
1592
1570
  value: options.value,
1593
1571
  tags: options.tags,
1594
1572
  ttl: options.ttl,
1573
+ metadata: options.metadata,
1595
1574
  });
1596
1575
  if (routed.routed && routed.ok) {
1597
- return { success: true, id: routed.id ?? '' };
1576
+ // #1065 surface the daemon's embedding metadata so the MCP
1577
+ // memory_store handler reports `hasEmbedding: true` on
1578
+ // daemon-routed writes (matching the bridge-direct shape).
1579
+ return { success: true, id: routed.id ?? '', embedding: routed.embedding };
1580
+ }
1581
+ // #1101 — daemon validated and rejected (4xx). Bridge-direct would
1582
+ // fail the same way; surface the daemon's error instead of silently
1583
+ // falling back.
1584
+ if (routed.routed && routed.ok === false) {
1585
+ return { success: false, id: '', error: routed.error ?? 'Daemon rejected store request' };
1598
1586
  }
1599
1587
  }
1600
1588
  catch (err) {
@@ -1611,7 +1599,7 @@ export async function storeEntry(options) {
1611
1599
  if (bridgeResult)
1612
1600
  return bridgeResult;
1613
1601
  }
1614
- // Fallback: raw sql.js
1602
+ // Fallback: direct node:sqlite write via the unified factory.
1615
1603
  const { key, value, namespace = 'default', generateEmbeddingFlag = true, tags = [], ttl, dbPath: customPath, upsert = false } = options;
1616
1604
  const dbPath = customPath || memoryDbPath(process.cwd());
1617
1605
  try {
@@ -1620,10 +1608,7 @@ export async function storeEntry(options) {
1620
1608
  }
1621
1609
  // Ensure schema has all required columns (migration for older DBs)
1622
1610
  await ensureSchemaColumns(dbPath);
1623
- const initSqlJs = (await mofloImport('sql.js')).default;
1624
- const SQL = await initSqlJs();
1625
- const fileBuffer = fs.readFileSync(dbPath);
1626
- const db = new SQL.Database(fileBuffer);
1611
+ const db = openDaemonDatabase(dbPath);
1627
1612
  const id = `entry_${Date.now()}_${Math.random().toString(36).substring(7)}`;
1628
1613
  const now = Date.now();
1629
1614
  // generateEmbedding() throws on embed failure; the outer try/catch returns
@@ -1654,7 +1639,7 @@ export async function storeEntry(options) {
1654
1639
  embeddingModel = embResult.model;
1655
1640
  }
1656
1641
  }
1657
- // Idempotency guard. By the time we reach the raw-sql.js fallback, an
1642
+ // Idempotency guard. By the time we reach the direct-write fallback, an
1658
1643
  // earlier write attempt — daemon route via `tryDaemonStore`, or bridge
1659
1644
  // via `bridgeStoreEntry` — may have already persisted this exact row to
1660
1645
  // disk. If a post-persist throw escaped the bridge's inner guards (#994,
@@ -1711,12 +1696,14 @@ export async function storeEntry(options) {
1711
1696
  embeddingDimensions,
1712
1697
  embeddingModel,
1713
1698
  tags.length > 0 ? JSON.stringify(tags) : null,
1714
- '{}',
1699
+ serialiseMetadata(options.metadata),
1715
1700
  now,
1716
1701
  now,
1717
1702
  ttl ? now + (ttl * 1000) : null
1718
1703
  ]);
1719
- atomicWriteFileSync(dbPath, db.export());
1704
+ // node:sqlite + WAL persisted that INSERT on commit — the sql.js
1705
+ // whole-file `db.export()` + atomicWriteFileSync that lived here was
1706
+ // the multi-writer clobber vector epic #1078 killed structurally.
1720
1707
  // Query exact stats while DB is still open. `missing` is the active-rows-
1721
1708
  // with-NULL-embedding count, surfaced via vector-stats.json so the
1722
1709
  // statusline can warn on coverage holes (#648 / #649).
@@ -1779,10 +1766,42 @@ export async function storeEntries(items, dbPath) {
1779
1766
  return out;
1780
1767
  }
1781
1768
  /**
1782
- * Search entries using sql.js with vector similarity
1783
- * Uses HNSW index for 150x faster search when available
1769
+ * Search entries via node:sqlite with vector similarity.
1770
+ * Uses HNSW index for 150x faster search when available.
1784
1771
  */
1785
1772
  export async function searchEntries(options) {
1773
+ // #1058 — read-side routing preamble. When a daemon is reachable AND we're
1774
+ // not the daemon ourselves AND no custom dbPath was supplied, route the
1775
+ // search through the daemon's HTTP RPC so callers see its authoritative,
1776
+ // up-to-the-write state. Without this, a non-daemon process queries its
1777
+ // own bridge's sql.js snapshot loaded at process-start and never sees
1778
+ // anything the daemon has written since (epic #1054 silent-drop).
1779
+ if (!options.dbPath
1780
+ && process.env.MOFLO_IS_DAEMON !== '1'
1781
+ && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
1782
+ try {
1783
+ const routed = await tryDaemonSearch({
1784
+ query: options.query,
1785
+ namespace: options.namespace,
1786
+ limit: options.limit,
1787
+ threshold: options.threshold,
1788
+ });
1789
+ if (routed.routed && routed.data) {
1790
+ return {
1791
+ success: true,
1792
+ results: routed.data.results,
1793
+ searchTime: routed.data.searchTime ?? 0,
1794
+ };
1795
+ }
1796
+ // #1101 — daemon rejected query (4xx); propagate instead of falling back.
1797
+ if (routed.routed && routed.error) {
1798
+ return { success: false, results: [], searchTime: 0, error: routed.error };
1799
+ }
1800
+ }
1801
+ catch (err) {
1802
+ logRoutingFault(err);
1803
+ }
1804
+ }
1786
1805
  // ADR-053: Try AgentDB v3 bridge first
1787
1806
  const bridge = await getBridge();
1788
1807
  if (bridge) {
@@ -1790,7 +1809,7 @@ export async function searchEntries(options) {
1790
1809
  if (bridgeResult)
1791
1810
  return bridgeResult;
1792
1811
  }
1793
- // Fallback: raw sql.js
1812
+ // Fallback: direct node:sqlite write via the unified factory.
1794
1813
  const { query, namespace = 'default', limit = 10, threshold = 0.3, dbPath: customPath } = options;
1795
1814
  const dbPath = customPath || memoryDbPath(process.cwd());
1796
1815
  const startTime = Date.now();
@@ -1814,11 +1833,8 @@ export async function searchEntries(options) {
1814
1833
  searchTime: Date.now() - startTime
1815
1834
  };
1816
1835
  }
1817
- // Fall back to brute-force SQLite search
1818
- const initSqlJs = (await mofloImport('sql.js')).default;
1819
- const SQL = await initSqlJs();
1820
- const fileBuffer = fs.readFileSync(dbPath);
1821
- const db = new SQL.Database(fileBuffer);
1836
+ // Fall back to brute-force SQLite search via the unified factory.
1837
+ const db = openDaemonDatabase(dbPath);
1822
1838
  // Get entries with embeddings
1823
1839
  const entries = db.exec(`
1824
1840
  SELECT id, key, namespace, content, metadata, embedding
@@ -1902,6 +1918,28 @@ function cosineSim(a, b) {
1902
1918
  * List all entries from the memory database
1903
1919
  */
1904
1920
  export async function listEntries(options) {
1921
+ // #1058 — read-side routing preamble (mirrors searchEntries/getEntry).
1922
+ if (!options.dbPath
1923
+ && process.env.MOFLO_IS_DAEMON !== '1'
1924
+ && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
1925
+ try {
1926
+ const routed = await tryDaemonList({
1927
+ namespace: options.namespace,
1928
+ limit: options.limit,
1929
+ offset: options.offset,
1930
+ });
1931
+ if (routed.routed && routed.data) {
1932
+ return { success: true, entries: routed.data.entries, total: routed.data.total };
1933
+ }
1934
+ // #1101 — daemon rejected list args (4xx); propagate.
1935
+ if (routed.routed && routed.error) {
1936
+ return { success: false, entries: [], total: 0, error: routed.error };
1937
+ }
1938
+ }
1939
+ catch (err) {
1940
+ logRoutingFault(err);
1941
+ }
1942
+ }
1905
1943
  // ADR-053: Try AgentDB v3 bridge first
1906
1944
  const bridge = await getBridge();
1907
1945
  if (bridge) {
@@ -1909,7 +1947,7 @@ export async function listEntries(options) {
1909
1947
  if (bridgeResult)
1910
1948
  return bridgeResult;
1911
1949
  }
1912
- // Fallback: raw sql.js
1950
+ // Fallback: direct node:sqlite write via the unified factory.
1913
1951
  const { namespace, limit = 20, offset = 0, dbPath: customPath } = options;
1914
1952
  const dbPath = customPath || memoryDbPath(process.cwd());
1915
1953
  try {
@@ -1918,10 +1956,7 @@ export async function listEntries(options) {
1918
1956
  }
1919
1957
  // Ensure schema has all required columns (migration for older DBs)
1920
1958
  await ensureSchemaColumns(dbPath);
1921
- const initSqlJs = (await mofloImport('sql.js')).default;
1922
- const SQL = await initSqlJs();
1923
- const fileBuffer = fs.readFileSync(dbPath);
1924
- const db = new SQL.Database(fileBuffer);
1959
+ const db = openDaemonDatabase(dbPath);
1925
1960
  // Get total count
1926
1961
  const countQuery = namespace
1927
1962
  ? `SELECT COUNT(*) as cnt FROM memory_entries WHERE status = 'active' AND namespace = '${namespace.replace(/'/g, "''")}'`
@@ -1970,6 +2005,27 @@ export async function listEntries(options) {
1970
2005
  * Get a specific entry from the memory database
1971
2006
  */
1972
2007
  export async function getEntry(options) {
2008
+ // #1058 — read-side routing preamble (mirrors searchEntries/listEntries).
2009
+ if (!options.dbPath
2010
+ && process.env.MOFLO_IS_DAEMON !== '1'
2011
+ && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
2012
+ try {
2013
+ const routed = await tryDaemonGet({
2014
+ namespace: options.namespace ?? 'default',
2015
+ key: options.key,
2016
+ });
2017
+ if (routed.routed && routed.data) {
2018
+ return { success: true, found: routed.data.found, entry: routed.data.entry };
2019
+ }
2020
+ // #1101 — daemon rejected get args (4xx); propagate.
2021
+ if (routed.routed && routed.error) {
2022
+ return { success: false, found: false, error: routed.error };
2023
+ }
2024
+ }
2025
+ catch (err) {
2026
+ logRoutingFault(err);
2027
+ }
2028
+ }
1973
2029
  // ADR-053: Try AgentDB v3 bridge first
1974
2030
  const bridge = await getBridge();
1975
2031
  if (bridge) {
@@ -1977,7 +2033,7 @@ export async function getEntry(options) {
1977
2033
  if (bridgeResult)
1978
2034
  return bridgeResult;
1979
2035
  }
1980
- // Fallback: raw sql.js
2036
+ // Fallback: direct node:sqlite write via the unified factory.
1981
2037
  const { key, namespace = 'default', dbPath: customPath } = options;
1982
2038
  const dbPath = customPath || memoryDbPath(process.cwd());
1983
2039
  try {
@@ -1986,10 +2042,7 @@ export async function getEntry(options) {
1986
2042
  }
1987
2043
  // Ensure schema has all required columns (migration for older DBs)
1988
2044
  await ensureSchemaColumns(dbPath);
1989
- const initSqlJs = (await mofloImport('sql.js')).default;
1990
- const SQL = await initSqlJs();
1991
- const fileBuffer = fs.readFileSync(dbPath);
1992
- const db = new SQL.Database(fileBuffer);
2045
+ const db = openDaemonDatabase(dbPath);
1993
2046
  // Find entry by key
1994
2047
  const result = db.exec(`
1995
2048
  SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, tags, metadata
@@ -2004,13 +2057,15 @@ export async function getEntry(options) {
2004
2057
  return { success: true, found: false };
2005
2058
  }
2006
2059
  const [id, entryKey, ns, content, embedding, accessCount, createdAt, updatedAt, tagsJson, metadataJson] = result[0].values[0];
2007
- // Update access count
2008
- db.run(`
2009
- UPDATE memory_entries
2010
- SET access_count = access_count + 1, last_accessed_at = strftime('%s', 'now') * 1000
2011
- WHERE id = '${String(id).replace(/'/g, "''")}'
2012
- `);
2013
- atomicWriteFileSync(dbPath, db.export());
2060
+ // #1058: previously this path issued `UPDATE memory_entries SET access_count = ...`
2061
+ // followed by `atomicWriteFileSync(dbPath, db.export())` — a read that
2062
+ // dumped the entire DB snapshot back to disk just to bump access_count.
2063
+ // Any write by another process between this function's readFileSync and
2064
+ // the writeback was clobbered (read-side writeback-clobber). Access_count
2065
+ // is observability, not correctness — drop the writeback. The caller's
2066
+ // return value reports the in-memory incremented count so existing
2067
+ // surfaces aren't disturbed; persistence of the counter is deferred to a
2068
+ // future controller-table refactor (out of scope).
2014
2069
  db.close();
2015
2070
  let tags = [];
2016
2071
  if (tagsJson) {
@@ -2053,7 +2108,7 @@ export async function getEntry(options) {
2053
2108
  export async function deleteEntry(options) {
2054
2109
  // #981 — single-writer routing for deletes. Same gates as storeEntry:
2055
2110
  // not the daemon, no custom dbPath, routing not opted out. Failure paths
2056
- // fall through to the existing bridge / raw-sql.js logic below.
2111
+ // fall through to the existing bridge / direct-write logic below.
2057
2112
  if (!options.dbPath
2058
2113
  && process.env.MOFLO_IS_DAEMON !== '1'
2059
2114
  && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
@@ -2074,6 +2129,17 @@ export async function deleteEntry(options) {
2074
2129
  remainingEntries: 0,
2075
2130
  };
2076
2131
  }
2132
+ // #1101 — daemon rejected delete args (4xx); propagate.
2133
+ if (routed.routed && routed.ok === false) {
2134
+ return {
2135
+ success: false,
2136
+ deleted: false,
2137
+ key: options.key,
2138
+ namespace: options.namespace ?? 'default',
2139
+ remainingEntries: 0,
2140
+ error: routed.error ?? 'Daemon rejected delete request',
2141
+ };
2142
+ }
2077
2143
  }
2078
2144
  catch (err) {
2079
2145
  logRoutingFault(err);
@@ -2086,7 +2152,7 @@ export async function deleteEntry(options) {
2086
2152
  if (bridgeResult)
2087
2153
  return bridgeResult;
2088
2154
  }
2089
- // Fallback: raw sql.js
2155
+ // Fallback: direct node:sqlite write via the unified factory.
2090
2156
  const { key, namespace = 'default', dbPath: customPath } = options;
2091
2157
  const dbPath = customPath || memoryDbPath(process.cwd());
2092
2158
  try {
@@ -2102,10 +2168,7 @@ export async function deleteEntry(options) {
2102
2168
  }
2103
2169
  // Ensure schema has all required columns (migration for older DBs)
2104
2170
  await ensureSchemaColumns(dbPath);
2105
- const initSqlJs = (await mofloImport('sql.js')).default;
2106
- const SQL = await initSqlJs();
2107
- const fileBuffer = fs.readFileSync(dbPath);
2108
- const db = new SQL.Database(fileBuffer);
2171
+ const db = openDaemonDatabase(dbPath);
2109
2172
  // Check if entry exists first
2110
2173
  const checkResult = db.exec(`
2111
2174
  SELECT id FROM memory_entries
@@ -2135,7 +2198,7 @@ export async function deleteEntry(options) {
2135
2198
  // Get remaining count
2136
2199
  const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
2137
2200
  const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
2138
- atomicWriteFileSync(dbPath, db.export());
2201
+ // WAL persisted the DELETE incrementally — no whole-file dump needed.
2139
2202
  db.close();
2140
2203
  return {
2141
2204
  success: true,
@@ -2166,10 +2229,7 @@ export async function getNamespaceCounts(dbPath) {
2166
2229
  if (!fs.existsSync(resolvedPath)) {
2167
2230
  return { namespaces: {}, total: 0 };
2168
2231
  }
2169
- const initSqlJs = (await mofloImport('sql.js')).default;
2170
- const SQL = await initSqlJs();
2171
- const fileBuffer = fs.readFileSync(resolvedPath);
2172
- const db = new SQL.Database(fileBuffer);
2232
+ const db = openDaemonDatabase(resolvedPath);
2173
2233
  const result = db.exec("SELECT namespace, COUNT(*) as cnt FROM memory_entries WHERE status = 'active' GROUP BY namespace ORDER BY cnt DESC");
2174
2234
  db.close();
2175
2235
  const namespaces = {};