moflo 4.10.24 → 4.10.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/.claude/guidance/shipped/moflo-yaml-reference.md +5 -5
  2. package/.claude/skills/meditate/SKILL.md +2 -2
  3. package/bin/cli-hooks/statusline.js +12 -8
  4. package/bin/cli.js +1 -1
  5. package/bin/hooks.mjs +1 -1
  6. package/bin/lib/meditate.mjs +1 -0
  7. package/bin/lib/pii-scrub.mjs +2 -5
  8. package/dist/src/cli/commands/daemon.js +4 -3
  9. package/dist/src/cli/commands/doctor-checks-deep.js +2 -2
  10. package/dist/src/cli/commands/doctor-checks-swarm.js +122 -13
  11. package/dist/src/cli/commands/hooks.js +7 -23
  12. package/dist/src/cli/commands/init.js +1 -1
  13. package/dist/src/cli/commands/mcp.js +2 -1
  14. package/dist/src/cli/commands/session.js +1 -1
  15. package/dist/src/cli/commands/start.js +1 -1
  16. package/dist/src/cli/commands/status.js +1 -1
  17. package/dist/src/cli/commands/task.js +1 -1
  18. package/dist/src/cli/commands/update.js +12 -12
  19. package/dist/src/cli/guidance/analyzer.js +3 -3
  20. package/dist/src/cli/guidance/gates.js +1 -1
  21. package/dist/src/cli/guidance/hooks.js +1 -1
  22. package/dist/src/cli/guidance/meta-governance.js +1 -1
  23. package/dist/src/cli/hooks/index.js +1 -1
  24. package/dist/src/cli/hooks/reasoningbank/guidance-provider.js +1 -1
  25. package/dist/src/cli/hooks/workers/index.js +1 -1
  26. package/dist/src/cli/hooks/workers/session-hook.js +0 -40
  27. package/dist/src/cli/index.js +2 -2
  28. package/dist/src/cli/init/executor.js +36 -20
  29. package/dist/src/cli/init/mcp-generator.js +10 -8
  30. package/dist/src/cli/init/settings-generator.js +10 -7
  31. package/dist/src/cli/init/types.js +2 -2
  32. package/dist/src/cli/mcp-server.js +2 -1
  33. package/dist/src/cli/memory/bridge-loader.js +42 -0
  34. package/dist/src/cli/memory/embedding-model.js +157 -0
  35. package/dist/src/cli/memory/entries-read.js +380 -0
  36. package/dist/src/cli/memory/entries-shared.js +73 -0
  37. package/dist/src/cli/memory/entries-write.js +384 -0
  38. package/dist/src/cli/memory/hnsw-singleton.js +242 -0
  39. package/dist/src/cli/memory/init.js +367 -0
  40. package/dist/src/cli/memory/learnings-overview.js +156 -0
  41. package/dist/src/cli/memory/memory-initializer.js +37 -2257
  42. package/dist/src/cli/memory/quantization.js +221 -0
  43. package/dist/src/cli/memory/schema.js +382 -0
  44. package/dist/src/cli/memory/verify.js +178 -0
  45. package/dist/src/cli/movector/index.js +1 -1
  46. package/dist/src/cli/plugins/store/discovery.js +9 -9
  47. package/dist/src/cli/{transfer/ipfs/client.js → plugins/store/ipfs-client.js} +4 -1
  48. package/dist/src/cli/plugins/tests/demo-plugin-store.js +1 -1
  49. package/dist/src/cli/plugins/tests/standalone-test.js +1 -1
  50. package/dist/src/cli/runtime/headless.js +5 -4
  51. package/dist/src/cli/scripts/publish-registry.js +6 -6
  52. package/dist/src/cli/services/daemon-dashboard.js +108 -7
  53. package/dist/src/cli/services/daemon-readiness.js +1 -1
  54. package/dist/src/cli/services/daemon-service.js +1 -1
  55. package/dist/src/cli/services/env-compat.js +29 -0
  56. package/dist/src/cli/services/hook-block-hash.js +5 -6
  57. package/dist/src/cli/services/registry-api.js +1 -1
  58. package/dist/src/cli/shared/core/config/loader.js +19 -11
  59. package/dist/src/cli/shared/events/example-usage.js +2 -2
  60. package/dist/src/cli/shared/events/index.js +1 -1
  61. package/dist/src/cli/shared/index.js +1 -1
  62. package/dist/src/cli/shared/mcp/index.js +1 -1
  63. package/dist/src/cli/shared/mcp/server.js +3 -3
  64. package/dist/src/cli/shared/plugin-interface.js +1 -1
  65. package/dist/src/cli/shared/plugins/index.js +1 -1
  66. package/dist/src/cli/shared/plugins/official/index.js +1 -1
  67. package/dist/src/cli/shared/security/index.js +1 -1
  68. package/dist/src/cli/shared/services/v3-progress.service.js +40 -29
  69. package/dist/src/cli/shared/types.js +1 -1
  70. package/dist/src/cli/swarm/coordination/swarm-hub.js +1 -1
  71. package/dist/src/cli/update/index.js +1 -1
  72. package/dist/src/cli/update/rate-limiter.js +3 -2
  73. package/dist/src/cli/version.js +1 -1
  74. package/package.json +2 -2
  75. package/dist/src/cli/commands/transfer-store.js +0 -428
  76. package/dist/src/cli/transfer/anonymization/index.js +0 -281
  77. package/dist/src/cli/transfer/deploy-seraphine.js +0 -205
  78. package/dist/src/cli/transfer/export.js +0 -113
  79. package/dist/src/cli/transfer/index.js +0 -31
  80. package/dist/src/cli/transfer/ipfs/upload.js +0 -411
  81. package/dist/src/cli/transfer/models/seraphine.js +0 -373
  82. package/dist/src/cli/transfer/serialization/cfp.js +0 -184
  83. package/dist/src/cli/transfer/storage/gcs.js +0 -242
  84. package/dist/src/cli/transfer/storage/index.js +0 -6
  85. package/dist/src/cli/transfer/store/discovery.js +0 -382
  86. package/dist/src/cli/transfer/store/download.js +0 -334
  87. package/dist/src/cli/transfer/store/index.js +0 -153
  88. package/dist/src/cli/transfer/store/publish.js +0 -294
  89. package/dist/src/cli/transfer/store/registry.js +0 -285
  90. package/dist/src/cli/transfer/store/search.js +0 -232
  91. package/dist/src/cli/transfer/store/tests/standalone-test.js +0 -190
  92. package/dist/src/cli/transfer/store/types.js +0 -6
  93. package/dist/src/cli/transfer/test-seraphine.js +0 -105
  94. package/dist/src/cli/transfer/tests/test-store.js +0 -214
  95. package/dist/src/cli/transfer/types.js +0 -6
@@ -0,0 +1,384 @@
1
+ /**
2
+ * Memory entry write path: store / bulk-store / delete.
3
+ *
4
+ * Extracted from `memory-initializer.ts` (#1203 decomposition). All writes
5
+ * follow the #981 single-writer routing preamble (route through the daemon's
6
+ * HTTP RPC when one is reachable), then fall back to the AgentDB v3 bridge,
7
+ * then to a direct node:sqlite write via the unified `openDaemonDatabase`
8
+ * factory.
9
+ *
10
+ * @module memory/entries-write
11
+ */
12
+ import * as fs from 'fs';
13
+ import { errorDetail } from '../shared/utils/error-detail.js';
14
+ import { memoryDbPath } from '../services/moflo-paths.js';
15
+ import { openDaemonDatabase } from './daemon-backend.js';
16
+ import { ensureSchemaColumns } from './schema.js';
17
+ import { generateEmbedding } from './embedding-model.js';
18
+ import { addToHNSWIndex } from './hnsw-singleton.js';
19
+ import { getBridge } from './bridge-loader.js';
20
+ import { tryDaemonStore, tryDaemonDelete } from './daemon-write-client.js';
21
+ import { EMBEDDING_MODEL_OPT_OUT, getBridgeEmbedder, isEphemeralNamespace } from './bridge-embedder.js';
22
+ import { toFloat32 } from './controllers/_shared.js';
23
+ import { serialiseMetadata } from './bridge-entries.js';
24
+ import { logRoutingFault, writeVectorStatsCache } from './entries-shared.js';
25
+ /**
26
+ * Store an entry directly via node:sqlite.
27
+ * This bypasses MCP and writes directly to the database.
28
+ */
29
+ export async function storeEntry(options) {
30
+ // Soft-redirect: `knowledge` is a deprecated alias for `learnings`. Writes
31
+ // are accepted but routed to learnings with provenance tags so future
32
+ // decay/prune treats user-forced entries as locked. Old consumer DBs that
33
+ // still have raw `knowledge` rows are migrated by
34
+ // bin/migrate-knowledge-to-learnings.mjs at session start.
35
+ if (options.namespace === 'knowledge') {
36
+ const incoming = options.tags ?? [];
37
+ const merged = new Set(incoming);
38
+ merged.add('source:user');
39
+ merged.add('locked');
40
+ options = { ...options, namespace: 'learnings', tags: [...merged] };
41
+ }
42
+ // #1203 — write-time provenance for the `learnings` namespace. Every learning
43
+ // must carry a `source:<origin>` tag so the Luminarium "Learnings" panel can
44
+ // show where each lesson came from. Specific writers stamp their own source
45
+ // (auto-meditate's distill pass → `source:auto-meditate`; the /meditate skill
46
+ // → `source:meditate-manual`; the knowledge redirect above → `source:user`).
47
+ // Anything else that lands in `learnings` without a source tag is an ad-hoc
48
+ // `memory_store`, tagged `source:manual`. Purely additive — rows that predate
49
+ // this simply lack the tag and fall into the panel's "legacy/unknown" bucket.
50
+ if (options.namespace === 'learnings') {
51
+ const tags = options.tags ?? [];
52
+ if (!tags.some((t) => typeof t === 'string' && t.startsWith('source:'))) {
53
+ options = { ...options, tags: [...tags, 'source:manual'] };
54
+ }
55
+ }
56
+ // #981 — single-writer routing. When an external daemon is reachable AND
57
+ // we're not the daemon ourselves AND no custom dbPath was supplied, route
58
+ // the write through the daemon's HTTP RPC so its in-memory handle stays
59
+ // authoritative. Any failure path falls through to the existing bridge /
60
+ // direct-write logic below — byte-identical behaviour to today.
61
+ if (!options.dbPath
62
+ && process.env.MOFLO_IS_DAEMON !== '1'
63
+ && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
64
+ try {
65
+ const routed = await tryDaemonStore({
66
+ namespace: options.namespace ?? 'default',
67
+ key: options.key,
68
+ value: options.value,
69
+ tags: options.tags,
70
+ ttl: options.ttl,
71
+ metadata: options.metadata,
72
+ });
73
+ if (routed.routed && routed.ok) {
74
+ // #1065 — surface the daemon's embedding metadata so the MCP
75
+ // memory_store handler reports `hasEmbedding: true` on
76
+ // daemon-routed writes (matching the bridge-direct shape).
77
+ return { success: true, id: routed.id ?? '', embedding: routed.embedding };
78
+ }
79
+ // #1101 — daemon validated and rejected (4xx). Bridge-direct would
80
+ // fail the same way; surface the daemon's error instead of silently
81
+ // falling back.
82
+ if (routed.routed && routed.ok === false) {
83
+ return { success: false, id: '', error: routed.error ?? 'Daemon rejected store request' };
84
+ }
85
+ }
86
+ catch (err) {
87
+ logRoutingFault(err);
88
+ }
89
+ }
90
+ // ADR-053: Try AgentDB v3 bridge first. The bridge calls
91
+ // refreshVectorStatsCache() itself (bridge-entries.ts:191) — a second
92
+ // write here was redundant and previously clobbered the correct count
93
+ // with 0 (#639).
94
+ const bridge = await getBridge();
95
+ if (bridge) {
96
+ const bridgeResult = await bridge.bridgeStoreEntry(options);
97
+ if (bridgeResult)
98
+ return bridgeResult;
99
+ }
100
+ // Fallback: direct node:sqlite write via the unified factory.
101
+ const { key, value, namespace = 'default', generateEmbeddingFlag = true, tags = [], ttl, dbPath: customPath, upsert = false } = options;
102
+ const dbPath = customPath || memoryDbPath(process.cwd());
103
+ try {
104
+ if (!fs.existsSync(dbPath)) {
105
+ return { success: false, id: '', error: 'Database not initialized. Run: flo memory init' };
106
+ }
107
+ // Ensure schema has all required columns (migration for older DBs)
108
+ await ensureSchemaColumns(dbPath);
109
+ const db = openDaemonDatabase(dbPath);
110
+ const id = `entry_${Date.now()}_${Math.random().toString(36).substring(7)}`;
111
+ const now = Date.now();
112
+ // generateEmbedding() throws on embed failure; the outer try/catch returns
113
+ // success:false rather than inserting a null-embedded row. Opt-out rows
114
+ // (generateEmbeddingFlag=false) are tagged EMBEDDING_MODEL_OPT_OUT — see
115
+ // the constant's docstring in bridge-embedder.ts for the rationale.
116
+ // Ephemeral namespaces (#729) skip embedding entirely AND tag model NULL.
117
+ let embeddingJson = null;
118
+ let embeddingDimensions = null;
119
+ let embeddingModel = EMBEDDING_MODEL_OPT_OUT;
120
+ const isEphemeralNs = isEphemeralNamespace(namespace);
121
+ if (isEphemeralNs) {
122
+ embeddingModel = null;
123
+ }
124
+ else if (generateEmbeddingFlag && value.length > 0) {
125
+ if (options.precomputedEmbedding) {
126
+ // Tag with the bridge embedder's canonical model so precomputed rows
127
+ // are indistinguishable from live single-embed rows downstream.
128
+ const vec = toFloat32(options.precomputedEmbedding);
129
+ embeddingJson = JSON.stringify(Array.from(vec));
130
+ embeddingDimensions = vec.length;
131
+ embeddingModel = getBridgeEmbedder().model;
132
+ }
133
+ else {
134
+ const embResult = await generateEmbedding(value);
135
+ embeddingJson = JSON.stringify(embResult.embedding);
136
+ embeddingDimensions = embResult.dimensions;
137
+ embeddingModel = embResult.model;
138
+ }
139
+ }
140
+ // Idempotency guard. By the time we reach the direct-write fallback, an
141
+ // earlier write attempt — daemon route via `tryDaemonStore`, or bridge
142
+ // via `bridgeStoreEntry` — may have already persisted this exact row to
143
+ // disk. If a post-persist throw escaped the bridge's inner guards (#994,
144
+ // #982), `bridgeStoreEntry` returned null and we landed here. Re-running
145
+ // a plain INSERT would then trip the UNIQUE constraint on `(namespace,
146
+ // key)` and surface as `exit 1` even though the data is durable on disk
147
+ // — exactly the cascade described in `bridge-entries.ts:205`. If the
148
+ // existing row matches the value the caller asked us to write, treat
149
+ // this as a successful no-op and propagate the existing id instead of
150
+ // re-inserting. If the content differs, fall through to INSERT — the
151
+ // UNIQUE error is then a real "key already taken with other content"
152
+ // signal that the caller deserves to see.
153
+ if (!upsert) {
154
+ let existingRow = null;
155
+ const probe = db.prepare(`SELECT id, content FROM memory_entries WHERE namespace = ? AND key = ? AND status = 'active' LIMIT 1`);
156
+ try {
157
+ probe.bind([namespace, key]);
158
+ if (probe.step()) {
159
+ existingRow = probe.getAsObject();
160
+ }
161
+ }
162
+ finally {
163
+ probe.free();
164
+ }
165
+ if (existingRow && existingRow.content === value) {
166
+ db.close();
167
+ return {
168
+ success: true,
169
+ id: String(existingRow.id),
170
+ embedding: embeddingJson
171
+ ? { dimensions: embeddingDimensions, model: embeddingModel }
172
+ : undefined,
173
+ };
174
+ }
175
+ }
176
+ // Insert or update entry (upsert mode uses REPLACE)
177
+ const insertSql = upsert
178
+ ? `INSERT OR REPLACE INTO memory_entries (
179
+ id, key, namespace, content, type,
180
+ embedding, embedding_dimensions, embedding_model,
181
+ tags, metadata, created_at, updated_at, expires_at, status
182
+ ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`
183
+ : `INSERT INTO memory_entries (
184
+ id, key, namespace, content, type,
185
+ embedding, embedding_dimensions, embedding_model,
186
+ tags, metadata, created_at, updated_at, expires_at, status
187
+ ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`;
188
+ db.run(insertSql, [
189
+ id,
190
+ key,
191
+ namespace,
192
+ value,
193
+ embeddingJson,
194
+ embeddingDimensions,
195
+ embeddingModel,
196
+ tags.length > 0 ? JSON.stringify(tags) : null,
197
+ serialiseMetadata(options.metadata),
198
+ now,
199
+ now,
200
+ ttl ? now + (ttl * 1000) : null
201
+ ]);
202
+ // node:sqlite + WAL persisted that INSERT on commit — the sql.js
203
+ // whole-file `db.export()` + atomicWriteFileSync that lived here was
204
+ // the multi-writer clobber vector epic #1078 killed structurally.
205
+ // Query exact stats while DB is still open. `missing` is the active-rows-
206
+ // with-NULL-embedding count, surfaced via vector-stats.json so the
207
+ // statusline can warn on coverage holes (#648 / #649).
208
+ let vecCount = 0, nsCount = 0, missingCount = 0;
209
+ try {
210
+ const vc = db.exec("SELECT COUNT(*) FROM memory_entries WHERE status='active' AND embedding IS NOT NULL");
211
+ vecCount = vc[0]?.values?.[0]?.[0] ?? 0;
212
+ const nc = db.exec("SELECT COUNT(DISTINCT namespace) FROM memory_entries WHERE status='active'");
213
+ nsCount = nc[0]?.values?.[0]?.[0] ?? 0;
214
+ const mc = db.exec("SELECT COUNT(*) FROM memory_entries WHERE status='active' AND embedding IS NULL");
215
+ missingCount = mc[0]?.values?.[0]?.[0] ?? 0;
216
+ }
217
+ catch { /* table may not have status column in older DBs */ }
218
+ db.close();
219
+ // Add to HNSW index for faster future searches
220
+ if (embeddingJson) {
221
+ const embResult = JSON.parse(embeddingJson);
222
+ await addToHNSWIndex(id, embResult, {
223
+ id,
224
+ key,
225
+ namespace,
226
+ content: value
227
+ });
228
+ }
229
+ // Update statusline cache with exact counts
230
+ writeVectorStatsCache(dbPath, { vectorCount: vecCount, namespaces: nsCount, missing: missingCount });
231
+ return {
232
+ success: true,
233
+ id,
234
+ embedding: embeddingJson ? { dimensions: embeddingDimensions, model: embeddingModel } : undefined
235
+ };
236
+ }
237
+ catch (error) {
238
+ return {
239
+ success: false,
240
+ id: '',
241
+ error: errorDetail(error)
242
+ };
243
+ }
244
+ }
245
+ /**
246
+ * Bulk-store entries — batches writes through the bridge in a single
247
+ * persist-once transaction. Falls back to sequential `storeEntry()` calls
248
+ * (each persisting independently) when the bridge is unavailable.
249
+ */
250
+ export async function storeEntries(items, dbPath) {
251
+ if (items.length === 0)
252
+ return [];
253
+ const bridge = await getBridge();
254
+ if (bridge && typeof bridge.bridgeStoreEntries === 'function') {
255
+ const bridgeResult = await bridge.bridgeStoreEntries(items, dbPath);
256
+ if (bridgeResult)
257
+ return bridgeResult;
258
+ }
259
+ // Fallback: sequential single-entry writes (each persists). Slow but correct.
260
+ const out = [];
261
+ for (const item of items) {
262
+ out.push(await storeEntry({ ...item, dbPath }));
263
+ }
264
+ return out;
265
+ }
266
+ /**
267
+ * Delete a memory entry by key and namespace
268
+ * Issue #980: Properly supports namespaced entries
269
+ */
270
+ export async function deleteEntry(options) {
271
+ // #981 — single-writer routing for deletes. Same gates as storeEntry:
272
+ // not the daemon, no custom dbPath, routing not opted out. Failure paths
273
+ // fall through to the existing bridge / direct-write logic below.
274
+ if (!options.dbPath
275
+ && process.env.MOFLO_IS_DAEMON !== '1'
276
+ && process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
277
+ try {
278
+ const routed = await tryDaemonDelete({
279
+ namespace: options.namespace ?? 'default',
280
+ key: options.key,
281
+ });
282
+ if (routed.routed && routed.ok) {
283
+ return {
284
+ success: true,
285
+ deleted: routed.deleted ?? true,
286
+ key: options.key,
287
+ namespace: options.namespace ?? 'default',
288
+ // Daemon doesn't surface remainingEntries; callers that depend on
289
+ // this value (the `flo memory delete` CLI) read it from a
290
+ // subsequent stat query, not this return shape.
291
+ remainingEntries: 0,
292
+ };
293
+ }
294
+ // #1101 — daemon rejected delete args (4xx); propagate.
295
+ if (routed.routed && routed.ok === false) {
296
+ return {
297
+ success: false,
298
+ deleted: false,
299
+ key: options.key,
300
+ namespace: options.namespace ?? 'default',
301
+ remainingEntries: 0,
302
+ error: routed.error ?? 'Daemon rejected delete request',
303
+ };
304
+ }
305
+ }
306
+ catch (err) {
307
+ logRoutingFault(err);
308
+ }
309
+ }
310
+ // ADR-053: Try AgentDB v3 bridge first
311
+ const bridge = await getBridge();
312
+ if (bridge) {
313
+ const bridgeResult = await bridge.bridgeDeleteEntry(options);
314
+ if (bridgeResult)
315
+ return bridgeResult;
316
+ }
317
+ // Fallback: direct node:sqlite write via the unified factory.
318
+ const { key, namespace = 'default', dbPath: customPath } = options;
319
+ const dbPath = customPath || memoryDbPath(process.cwd());
320
+ try {
321
+ if (!fs.existsSync(dbPath)) {
322
+ return {
323
+ success: false,
324
+ deleted: false,
325
+ key,
326
+ namespace,
327
+ remainingEntries: 0,
328
+ error: 'Database not found'
329
+ };
330
+ }
331
+ // Ensure schema has all required columns (migration for older DBs)
332
+ await ensureSchemaColumns(dbPath);
333
+ const db = openDaemonDatabase(dbPath);
334
+ // Check if entry exists first
335
+ const checkResult = db.exec(`
336
+ SELECT id FROM memory_entries
337
+ WHERE status = 'active'
338
+ AND key = '${key.replace(/'/g, "''")}'
339
+ AND namespace = '${namespace.replace(/'/g, "''")}'
340
+ LIMIT 1
341
+ `);
342
+ if (!checkResult[0]?.values?.[0]) {
343
+ // Get remaining count before closing
344
+ const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
345
+ const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
346
+ db.close();
347
+ return {
348
+ success: true,
349
+ deleted: false,
350
+ key,
351
+ namespace,
352
+ remainingEntries,
353
+ error: `Key '${key}' not found in namespace '${namespace}'`
354
+ };
355
+ }
356
+ // Hard-delete the entry. Soft-delete was retired in story #728: tombstones
357
+ // were write-only (no code ever restored from status='deleted') and bloated
358
+ // the DB indefinitely.
359
+ db.run(`DELETE FROM memory_entries WHERE key = ? AND namespace = ? AND status = 'active'`, [key, namespace]);
360
+ // Get remaining count
361
+ const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
362
+ const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
363
+ // WAL persisted the DELETE incrementally — no whole-file dump needed.
364
+ db.close();
365
+ return {
366
+ success: true,
367
+ deleted: true,
368
+ key,
369
+ namespace,
370
+ remainingEntries
371
+ };
372
+ }
373
+ catch (error) {
374
+ return {
375
+ success: false,
376
+ deleted: false,
377
+ key,
378
+ namespace,
379
+ remainingEntries: 0,
380
+ error: errorDetail(error)
381
+ };
382
+ }
383
+ }
384
+ //# sourceMappingURL=entries-write.js.map
@@ -0,0 +1,242 @@
1
+ /**
2
+ * Process-wide HNSW vector-index singleton (150x faster vector search).
3
+ *
4
+ * Extracted from `memory-initializer.ts` (#1203 decomposition). Owns the
5
+ * lazy HNSW index built from the SQLite `embedding` column (or a binary
6
+ * sidecar), plus add/search/status/clear. Uses the pure-TS {@link HnswLite}
7
+ * implementation — no native dependencies.
8
+ *
9
+ * Distinct from `hnsw-index.ts` (the standalone HNSWConfig/HNSWStats
10
+ * implementation): this module is the singleton the memory-CRUD path wires
11
+ * into via `getHNSWIndex` / `addToHNSWIndex` / `searchHNSWIndex`.
12
+ *
13
+ * @module memory/hnsw-singleton
14
+ */
15
+ import * as fs from 'fs';
16
+ import * as path from 'path';
17
+ import { HnswLite } from './hnsw-lite.js';
18
+ import { tryLoadHnswSidecar } from './hnsw-persistence.js';
19
+ import { parseEmbeddingJson } from './controllers/_shared.js';
20
+ import { memoryDbPath } from '../services/moflo-paths.js';
21
+ import { openDaemonDatabase } from './daemon-backend.js';
22
+ import { getBridge, isBridgeLoaded } from './bridge-loader.js';
23
+ let hnswIndex = null;
24
+ let hnswInitializing = false;
25
+ /**
26
+ * Get or create the HNSW index singleton
27
+ * Lazily initializes from SQLite data on first use
28
+ */
29
+ export async function getHNSWIndex(options) {
30
+ const dimensions = options?.dimensions ?? 384;
31
+ // Return existing index if already initialized
32
+ if (hnswIndex?.initialized && !options?.forceRebuild) {
33
+ return hnswIndex;
34
+ }
35
+ // Prevent concurrent initialization
36
+ if (hnswInitializing) {
37
+ // Wait for initialization to complete
38
+ while (hnswInitializing) {
39
+ await new Promise(resolve => setTimeout(resolve, 10));
40
+ }
41
+ return hnswIndex;
42
+ }
43
+ hnswInitializing = true;
44
+ try {
45
+ // Use HnswLite pure TS implementation (no native dependencies).
46
+ // Persistent storage paths — colocated with the canonical memory DB.
47
+ const dbPath = options?.dbPath || memoryDbPath(process.cwd());
48
+ const dbDir = path.dirname(dbPath);
49
+ if (!fs.existsSync(dbDir)) {
50
+ fs.mkdirSync(dbDir, { recursive: true });
51
+ }
52
+ const projectRoot = path.dirname(dbDir);
53
+ // Try the binary sidecar first — graph + neighbors round-trip exactly,
54
+ // so the cold-start cost drops to one readFileSync + slice. Fall back
55
+ // to SQL-rebuild only when the sidecar is missing or malformed.
56
+ const loadedFromSidecar = options?.forceRebuild ? null : tryLoadHnswSidecar(projectRoot);
57
+ const hnsw = loadedFromSidecar ?? new HnswLite(dimensions, 16, 200, 'cosine');
58
+ const sidecarLoaded = loadedFromSidecar !== null;
59
+ const db = {
60
+ insert: async (entry) => {
61
+ hnsw.add(entry.id, entry.vector);
62
+ },
63
+ search: async (query) => {
64
+ return hnsw.search(query.vector, query.k);
65
+ },
66
+ len: async () => hnsw.size,
67
+ };
68
+ const entries = new Map();
69
+ hnswIndex = {
70
+ db,
71
+ entries,
72
+ dimensions,
73
+ initialized: false
74
+ };
75
+ // Always populate the entries metadata from SQL — `key/namespace/content`
76
+ // is the source of truth there, and the sidecar only stores vectors +
77
+ // adjacency. When the sidecar IS loaded we skip the per-row JSON.parse
78
+ // of the embedding column, which is the expensive part on a populated
79
+ // consumer DB.
80
+ const SELECT_WITH_EMBEDDING = `id, key, namespace, content, metadata, embedding`;
81
+ const SELECT_METADATA_ONLY = `id, key, namespace, content, metadata`;
82
+ if (fs.existsSync(dbPath)) {
83
+ try {
84
+ const sqlDb = openDaemonDatabase(dbPath);
85
+ const cols = sidecarLoaded ? SELECT_METADATA_ONLY : SELECT_WITH_EMBEDDING;
86
+ const result = sqlDb.exec(`
87
+ SELECT ${cols}
88
+ FROM memory_entries
89
+ WHERE status = 'active' AND embedding IS NOT NULL
90
+ LIMIT 10000
91
+ `);
92
+ let parseSkipped = 0;
93
+ if (result[0]?.values) {
94
+ for (const row of result[0].values) {
95
+ // Column order matches SELECT_WITH_EMBEDDING / SELECT_METADATA_ONLY.
96
+ // When sidecar is loaded, embeddingJson is undefined (column absent).
97
+ const [id, key, ns, content, metadataJson, embeddingJson] = row;
98
+ if (!sidecarLoaded) {
99
+ const vec = parseEmbeddingJson(embeddingJson);
100
+ if (!vec) {
101
+ parseSkipped++;
102
+ continue;
103
+ }
104
+ await db.insert({ id: String(id), vector: vec });
105
+ }
106
+ hnswIndex.entries.set(String(id), {
107
+ id: String(id),
108
+ key: key || String(id),
109
+ namespace: ns || 'default',
110
+ content: content || '',
111
+ metadata: metadataJson || undefined
112
+ });
113
+ }
114
+ }
115
+ if (parseSkipped > 0) {
116
+ console.warn(`[memory-initializer] skipped ${parseSkipped} rows with malformed embeddings`);
117
+ }
118
+ sqlDb.close();
119
+ }
120
+ catch (err) {
121
+ console.warn(`[memory-initializer] SQL load failed, starting empty: ${err.message}`);
122
+ }
123
+ }
124
+ hnswIndex.initialized = true;
125
+ hnswInitializing = false;
126
+ return hnswIndex;
127
+ }
128
+ catch (err) {
129
+ console.warn(`[memory-initializer] getHNSWIndex failed: ${err.message}`);
130
+ hnswInitializing = false;
131
+ return null;
132
+ }
133
+ }
134
+ /**
135
+ * Add entry to HNSW index. Live-adds stay in-memory until the next
136
+ * `memory rebuild-index` run rebuilds the binary sidecar at
137
+ * `.moflo/hnsw.index`. The sql.js `embedding` column is the source of
138
+ * truth across process boundaries.
139
+ */
140
+ export async function addToHNSWIndex(id, embedding, entry) {
141
+ // ADR-053: Try AgentDB v3 bridge first
142
+ const bridge = await getBridge();
143
+ if (bridge) {
144
+ const bridgeResult = await bridge.bridgeAddToHNSW(id, embedding, entry);
145
+ if (bridgeResult === true)
146
+ return true;
147
+ }
148
+ const index = await getHNSWIndex({ dimensions: embedding.length });
149
+ if (!index)
150
+ return false;
151
+ try {
152
+ const vector = new Float32Array(embedding);
153
+ await index.db.insert({
154
+ id,
155
+ vector
156
+ });
157
+ index.entries.set(id, entry);
158
+ return true;
159
+ }
160
+ catch {
161
+ return false;
162
+ }
163
+ }
164
+ /**
165
+ * Search HNSW index (150x faster than brute-force)
166
+ * Returns results sorted by similarity (highest first)
167
+ */
168
+ export async function searchHNSWIndex(queryEmbedding, options) {
169
+ // ADR-053: Try AgentDB v3 bridge first
170
+ const bridge = await getBridge();
171
+ if (bridge) {
172
+ const bridgeResult = await bridge.bridgeSearchHNSW(queryEmbedding, options);
173
+ if (bridgeResult)
174
+ return bridgeResult;
175
+ }
176
+ const index = await getHNSWIndex({ dimensions: queryEmbedding.length });
177
+ if (!index)
178
+ return null;
179
+ try {
180
+ const vector = new Float32Array(queryEmbedding);
181
+ const k = options?.k ?? 10;
182
+ // HNSW search returns results with cosine distance (lower = more similar)
183
+ const results = await index.db.search({ vector, k: k * 2 }); // Get extra for filtering
184
+ const filtered = [];
185
+ for (const result of results) {
186
+ const entry = index.entries.get(result.id);
187
+ if (!entry)
188
+ continue;
189
+ // Filter by namespace if specified
190
+ if (options?.namespace && options.namespace !== 'all' && entry.namespace !== options.namespace) {
191
+ continue;
192
+ }
193
+ // Convert cosine distance to similarity score (1 - distance)
194
+ // Cosine distance: 0 = identical, 2 = opposite
195
+ const score = 1 - (result.score / 2);
196
+ filtered.push({
197
+ id: entry.id.substring(0, 12),
198
+ key: entry.key || entry.id.substring(0, 15),
199
+ content: entry.content.substring(0, 60) + (entry.content.length > 60 ? '...' : ''),
200
+ score,
201
+ namespace: entry.namespace,
202
+ metadata: entry.metadata
203
+ });
204
+ if (filtered.length >= k)
205
+ break;
206
+ }
207
+ // Sort by score descending (highest similarity first)
208
+ filtered.sort((a, b) => b.score - a.score);
209
+ return filtered;
210
+ }
211
+ catch {
212
+ return null;
213
+ }
214
+ }
215
+ /**
216
+ * Get HNSW index status
217
+ */
218
+ export function getHNSWStatus() {
219
+ // ADR-053: If bridge was previously loaded, report availability
220
+ if (isBridgeLoaded()) {
221
+ // Bridge is loaded — HNSW-equivalent is available via AgentDB v3
222
+ return {
223
+ available: true,
224
+ initialized: true,
225
+ entryCount: hnswIndex?.entries.size ?? 0,
226
+ dimensions: hnswIndex?.dimensions ?? 384
227
+ };
228
+ }
229
+ return {
230
+ available: hnswIndex !== null,
231
+ initialized: hnswIndex?.initialized ?? false,
232
+ entryCount: hnswIndex?.entries.size ?? 0,
233
+ dimensions: hnswIndex?.dimensions ?? 384
234
+ };
235
+ }
236
+ /**
237
+ * Clear the HNSW index (for rebuilding)
238
+ */
239
+ export function clearHNSWIndex() {
240
+ hnswIndex = null;
241
+ }
242
+ //# sourceMappingURL=hnsw-singleton.js.map