moflo 4.10.24 → 4.10.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-yaml-reference.md +5 -5
- package/.claude/skills/meditate/SKILL.md +2 -2
- package/bin/cli-hooks/statusline.js +12 -8
- package/bin/cli.js +1 -1
- package/bin/hooks.mjs +1 -1
- package/bin/lib/meditate.mjs +1 -0
- package/bin/lib/pii-scrub.mjs +2 -5
- package/dist/src/cli/commands/daemon.js +4 -3
- package/dist/src/cli/commands/doctor-checks-deep.js +2 -2
- package/dist/src/cli/commands/doctor-checks-swarm.js +122 -13
- package/dist/src/cli/commands/hooks.js +7 -23
- package/dist/src/cli/commands/init.js +1 -1
- package/dist/src/cli/commands/mcp.js +2 -1
- package/dist/src/cli/commands/session.js +1 -1
- package/dist/src/cli/commands/start.js +1 -1
- package/dist/src/cli/commands/status.js +1 -1
- package/dist/src/cli/commands/task.js +1 -1
- package/dist/src/cli/commands/update.js +12 -12
- package/dist/src/cli/guidance/analyzer.js +3 -3
- package/dist/src/cli/guidance/gates.js +1 -1
- package/dist/src/cli/guidance/hooks.js +1 -1
- package/dist/src/cli/guidance/meta-governance.js +1 -1
- package/dist/src/cli/hooks/index.js +1 -1
- package/dist/src/cli/hooks/reasoningbank/guidance-provider.js +1 -1
- package/dist/src/cli/hooks/workers/index.js +1 -1
- package/dist/src/cli/hooks/workers/session-hook.js +0 -40
- package/dist/src/cli/index.js +2 -2
- package/dist/src/cli/init/executor.js +36 -20
- package/dist/src/cli/init/mcp-generator.js +10 -8
- package/dist/src/cli/init/settings-generator.js +10 -7
- package/dist/src/cli/init/types.js +2 -2
- package/dist/src/cli/mcp-server.js +2 -1
- package/dist/src/cli/memory/bridge-loader.js +42 -0
- package/dist/src/cli/memory/embedding-model.js +157 -0
- package/dist/src/cli/memory/entries-read.js +380 -0
- package/dist/src/cli/memory/entries-shared.js +73 -0
- package/dist/src/cli/memory/entries-write.js +384 -0
- package/dist/src/cli/memory/hnsw-singleton.js +242 -0
- package/dist/src/cli/memory/init.js +367 -0
- package/dist/src/cli/memory/learnings-overview.js +156 -0
- package/dist/src/cli/memory/memory-initializer.js +37 -2257
- package/dist/src/cli/memory/quantization.js +221 -0
- package/dist/src/cli/memory/schema.js +382 -0
- package/dist/src/cli/memory/verify.js +178 -0
- package/dist/src/cli/movector/index.js +1 -1
- package/dist/src/cli/plugins/store/discovery.js +9 -9
- package/dist/src/cli/{transfer/ipfs/client.js → plugins/store/ipfs-client.js} +4 -1
- package/dist/src/cli/plugins/tests/demo-plugin-store.js +1 -1
- package/dist/src/cli/plugins/tests/standalone-test.js +1 -1
- package/dist/src/cli/runtime/headless.js +5 -4
- package/dist/src/cli/scripts/publish-registry.js +6 -6
- package/dist/src/cli/services/daemon-dashboard.js +108 -7
- package/dist/src/cli/services/daemon-readiness.js +1 -1
- package/dist/src/cli/services/daemon-service.js +1 -1
- package/dist/src/cli/services/env-compat.js +29 -0
- package/dist/src/cli/services/hook-block-hash.js +5 -6
- package/dist/src/cli/services/registry-api.js +1 -1
- package/dist/src/cli/shared/core/config/loader.js +19 -11
- package/dist/src/cli/shared/events/example-usage.js +2 -2
- package/dist/src/cli/shared/events/index.js +1 -1
- package/dist/src/cli/shared/index.js +1 -1
- package/dist/src/cli/shared/mcp/index.js +1 -1
- package/dist/src/cli/shared/mcp/server.js +3 -3
- package/dist/src/cli/shared/plugin-interface.js +1 -1
- package/dist/src/cli/shared/plugins/index.js +1 -1
- package/dist/src/cli/shared/plugins/official/index.js +1 -1
- package/dist/src/cli/shared/security/index.js +1 -1
- package/dist/src/cli/shared/services/v3-progress.service.js +40 -29
- package/dist/src/cli/shared/types.js +1 -1
- package/dist/src/cli/swarm/coordination/swarm-hub.js +1 -1
- package/dist/src/cli/update/index.js +1 -1
- package/dist/src/cli/update/rate-limiter.js +3 -2
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -2
- package/dist/src/cli/commands/transfer-store.js +0 -428
- package/dist/src/cli/transfer/anonymization/index.js +0 -281
- package/dist/src/cli/transfer/deploy-seraphine.js +0 -205
- package/dist/src/cli/transfer/export.js +0 -113
- package/dist/src/cli/transfer/index.js +0 -31
- package/dist/src/cli/transfer/ipfs/upload.js +0 -411
- package/dist/src/cli/transfer/models/seraphine.js +0 -373
- package/dist/src/cli/transfer/serialization/cfp.js +0 -184
- package/dist/src/cli/transfer/storage/gcs.js +0 -242
- package/dist/src/cli/transfer/storage/index.js +0 -6
- package/dist/src/cli/transfer/store/discovery.js +0 -382
- package/dist/src/cli/transfer/store/download.js +0 -334
- package/dist/src/cli/transfer/store/index.js +0 -153
- package/dist/src/cli/transfer/store/publish.js +0 -294
- package/dist/src/cli/transfer/store/registry.js +0 -285
- package/dist/src/cli/transfer/store/search.js +0 -232
- package/dist/src/cli/transfer/store/tests/standalone-test.js +0 -190
- package/dist/src/cli/transfer/store/types.js +0 -6
- package/dist/src/cli/transfer/test-seraphine.js +0 -105
- package/dist/src/cli/transfer/tests/test-store.js +0 -214
- package/dist/src/cli/transfer/types.js +0 -6
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory entry write path: store / bulk-store / delete.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `memory-initializer.ts` (#1203 decomposition). All writes
|
|
5
|
+
* follow the #981 single-writer routing preamble (route through the daemon's
|
|
6
|
+
* HTTP RPC when one is reachable), then fall back to the AgentDB v3 bridge,
|
|
7
|
+
* then to a direct node:sqlite write via the unified `openDaemonDatabase`
|
|
8
|
+
* factory.
|
|
9
|
+
*
|
|
10
|
+
* @module memory/entries-write
|
|
11
|
+
*/
|
|
12
|
+
import * as fs from 'fs';
|
|
13
|
+
import { errorDetail } from '../shared/utils/error-detail.js';
|
|
14
|
+
import { memoryDbPath } from '../services/moflo-paths.js';
|
|
15
|
+
import { openDaemonDatabase } from './daemon-backend.js';
|
|
16
|
+
import { ensureSchemaColumns } from './schema.js';
|
|
17
|
+
import { generateEmbedding } from './embedding-model.js';
|
|
18
|
+
import { addToHNSWIndex } from './hnsw-singleton.js';
|
|
19
|
+
import { getBridge } from './bridge-loader.js';
|
|
20
|
+
import { tryDaemonStore, tryDaemonDelete } from './daemon-write-client.js';
|
|
21
|
+
import { EMBEDDING_MODEL_OPT_OUT, getBridgeEmbedder, isEphemeralNamespace } from './bridge-embedder.js';
|
|
22
|
+
import { toFloat32 } from './controllers/_shared.js';
|
|
23
|
+
import { serialiseMetadata } from './bridge-entries.js';
|
|
24
|
+
import { logRoutingFault, writeVectorStatsCache } from './entries-shared.js';
|
|
25
|
+
/**
|
|
26
|
+
* Store an entry directly via node:sqlite.
|
|
27
|
+
* This bypasses MCP and writes directly to the database.
|
|
28
|
+
*/
|
|
29
|
+
export async function storeEntry(options) {
|
|
30
|
+
// Soft-redirect: `knowledge` is a deprecated alias for `learnings`. Writes
|
|
31
|
+
// are accepted but routed to learnings with provenance tags so future
|
|
32
|
+
// decay/prune treats user-forced entries as locked. Old consumer DBs that
|
|
33
|
+
// still have raw `knowledge` rows are migrated by
|
|
34
|
+
// bin/migrate-knowledge-to-learnings.mjs at session start.
|
|
35
|
+
if (options.namespace === 'knowledge') {
|
|
36
|
+
const incoming = options.tags ?? [];
|
|
37
|
+
const merged = new Set(incoming);
|
|
38
|
+
merged.add('source:user');
|
|
39
|
+
merged.add('locked');
|
|
40
|
+
options = { ...options, namespace: 'learnings', tags: [...merged] };
|
|
41
|
+
}
|
|
42
|
+
// #1203 — write-time provenance for the `learnings` namespace. Every learning
|
|
43
|
+
// must carry a `source:<origin>` tag so the Luminarium "Learnings" panel can
|
|
44
|
+
// show where each lesson came from. Specific writers stamp their own source
|
|
45
|
+
// (auto-meditate's distill pass → `source:auto-meditate`; the /meditate skill
|
|
46
|
+
// → `source:meditate-manual`; the knowledge redirect above → `source:user`).
|
|
47
|
+
// Anything else that lands in `learnings` without a source tag is an ad-hoc
|
|
48
|
+
// `memory_store`, tagged `source:manual`. Purely additive — rows that predate
|
|
49
|
+
// this simply lack the tag and fall into the panel's "legacy/unknown" bucket.
|
|
50
|
+
if (options.namespace === 'learnings') {
|
|
51
|
+
const tags = options.tags ?? [];
|
|
52
|
+
if (!tags.some((t) => typeof t === 'string' && t.startsWith('source:'))) {
|
|
53
|
+
options = { ...options, tags: [...tags, 'source:manual'] };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// #981 — single-writer routing. When an external daemon is reachable AND
|
|
57
|
+
// we're not the daemon ourselves AND no custom dbPath was supplied, route
|
|
58
|
+
// the write through the daemon's HTTP RPC so its in-memory handle stays
|
|
59
|
+
// authoritative. Any failure path falls through to the existing bridge /
|
|
60
|
+
// direct-write logic below — byte-identical behaviour to today.
|
|
61
|
+
if (!options.dbPath
|
|
62
|
+
&& process.env.MOFLO_IS_DAEMON !== '1'
|
|
63
|
+
&& process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
|
|
64
|
+
try {
|
|
65
|
+
const routed = await tryDaemonStore({
|
|
66
|
+
namespace: options.namespace ?? 'default',
|
|
67
|
+
key: options.key,
|
|
68
|
+
value: options.value,
|
|
69
|
+
tags: options.tags,
|
|
70
|
+
ttl: options.ttl,
|
|
71
|
+
metadata: options.metadata,
|
|
72
|
+
});
|
|
73
|
+
if (routed.routed && routed.ok) {
|
|
74
|
+
// #1065 — surface the daemon's embedding metadata so the MCP
|
|
75
|
+
// memory_store handler reports `hasEmbedding: true` on
|
|
76
|
+
// daemon-routed writes (matching the bridge-direct shape).
|
|
77
|
+
return { success: true, id: routed.id ?? '', embedding: routed.embedding };
|
|
78
|
+
}
|
|
79
|
+
// #1101 — daemon validated and rejected (4xx). Bridge-direct would
|
|
80
|
+
// fail the same way; surface the daemon's error instead of silently
|
|
81
|
+
// falling back.
|
|
82
|
+
if (routed.routed && routed.ok === false) {
|
|
83
|
+
return { success: false, id: '', error: routed.error ?? 'Daemon rejected store request' };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
logRoutingFault(err);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// ADR-053: Try AgentDB v3 bridge first. The bridge calls
|
|
91
|
+
// refreshVectorStatsCache() itself (bridge-entries.ts:191) — a second
|
|
92
|
+
// write here was redundant and previously clobbered the correct count
|
|
93
|
+
// with 0 (#639).
|
|
94
|
+
const bridge = await getBridge();
|
|
95
|
+
if (bridge) {
|
|
96
|
+
const bridgeResult = await bridge.bridgeStoreEntry(options);
|
|
97
|
+
if (bridgeResult)
|
|
98
|
+
return bridgeResult;
|
|
99
|
+
}
|
|
100
|
+
// Fallback: direct node:sqlite write via the unified factory.
|
|
101
|
+
const { key, value, namespace = 'default', generateEmbeddingFlag = true, tags = [], ttl, dbPath: customPath, upsert = false } = options;
|
|
102
|
+
const dbPath = customPath || memoryDbPath(process.cwd());
|
|
103
|
+
try {
|
|
104
|
+
if (!fs.existsSync(dbPath)) {
|
|
105
|
+
return { success: false, id: '', error: 'Database not initialized. Run: flo memory init' };
|
|
106
|
+
}
|
|
107
|
+
// Ensure schema has all required columns (migration for older DBs)
|
|
108
|
+
await ensureSchemaColumns(dbPath);
|
|
109
|
+
const db = openDaemonDatabase(dbPath);
|
|
110
|
+
const id = `entry_${Date.now()}_${Math.random().toString(36).substring(7)}`;
|
|
111
|
+
const now = Date.now();
|
|
112
|
+
// generateEmbedding() throws on embed failure; the outer try/catch returns
|
|
113
|
+
// success:false rather than inserting a null-embedded row. Opt-out rows
|
|
114
|
+
// (generateEmbeddingFlag=false) are tagged EMBEDDING_MODEL_OPT_OUT — see
|
|
115
|
+
// the constant's docstring in bridge-embedder.ts for the rationale.
|
|
116
|
+
// Ephemeral namespaces (#729) skip embedding entirely AND tag model NULL.
|
|
117
|
+
let embeddingJson = null;
|
|
118
|
+
let embeddingDimensions = null;
|
|
119
|
+
let embeddingModel = EMBEDDING_MODEL_OPT_OUT;
|
|
120
|
+
const isEphemeralNs = isEphemeralNamespace(namespace);
|
|
121
|
+
if (isEphemeralNs) {
|
|
122
|
+
embeddingModel = null;
|
|
123
|
+
}
|
|
124
|
+
else if (generateEmbeddingFlag && value.length > 0) {
|
|
125
|
+
if (options.precomputedEmbedding) {
|
|
126
|
+
// Tag with the bridge embedder's canonical model so precomputed rows
|
|
127
|
+
// are indistinguishable from live single-embed rows downstream.
|
|
128
|
+
const vec = toFloat32(options.precomputedEmbedding);
|
|
129
|
+
embeddingJson = JSON.stringify(Array.from(vec));
|
|
130
|
+
embeddingDimensions = vec.length;
|
|
131
|
+
embeddingModel = getBridgeEmbedder().model;
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
const embResult = await generateEmbedding(value);
|
|
135
|
+
embeddingJson = JSON.stringify(embResult.embedding);
|
|
136
|
+
embeddingDimensions = embResult.dimensions;
|
|
137
|
+
embeddingModel = embResult.model;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// Idempotency guard. By the time we reach the direct-write fallback, an
|
|
141
|
+
// earlier write attempt — daemon route via `tryDaemonStore`, or bridge
|
|
142
|
+
// via `bridgeStoreEntry` — may have already persisted this exact row to
|
|
143
|
+
// disk. If a post-persist throw escaped the bridge's inner guards (#994,
|
|
144
|
+
// #982), `bridgeStoreEntry` returned null and we landed here. Re-running
|
|
145
|
+
// a plain INSERT would then trip the UNIQUE constraint on `(namespace,
|
|
146
|
+
// key)` and surface as `exit 1` even though the data is durable on disk
|
|
147
|
+
// — exactly the cascade described in `bridge-entries.ts:205`. If the
|
|
148
|
+
// existing row matches the value the caller asked us to write, treat
|
|
149
|
+
// this as a successful no-op and propagate the existing id instead of
|
|
150
|
+
// re-inserting. If the content differs, fall through to INSERT — the
|
|
151
|
+
// UNIQUE error is then a real "key already taken with other content"
|
|
152
|
+
// signal that the caller deserves to see.
|
|
153
|
+
if (!upsert) {
|
|
154
|
+
let existingRow = null;
|
|
155
|
+
const probe = db.prepare(`SELECT id, content FROM memory_entries WHERE namespace = ? AND key = ? AND status = 'active' LIMIT 1`);
|
|
156
|
+
try {
|
|
157
|
+
probe.bind([namespace, key]);
|
|
158
|
+
if (probe.step()) {
|
|
159
|
+
existingRow = probe.getAsObject();
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
finally {
|
|
163
|
+
probe.free();
|
|
164
|
+
}
|
|
165
|
+
if (existingRow && existingRow.content === value) {
|
|
166
|
+
db.close();
|
|
167
|
+
return {
|
|
168
|
+
success: true,
|
|
169
|
+
id: String(existingRow.id),
|
|
170
|
+
embedding: embeddingJson
|
|
171
|
+
? { dimensions: embeddingDimensions, model: embeddingModel }
|
|
172
|
+
: undefined,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// Insert or update entry (upsert mode uses REPLACE)
|
|
177
|
+
const insertSql = upsert
|
|
178
|
+
? `INSERT OR REPLACE INTO memory_entries (
|
|
179
|
+
id, key, namespace, content, type,
|
|
180
|
+
embedding, embedding_dimensions, embedding_model,
|
|
181
|
+
tags, metadata, created_at, updated_at, expires_at, status
|
|
182
|
+
) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`
|
|
183
|
+
: `INSERT INTO memory_entries (
|
|
184
|
+
id, key, namespace, content, type,
|
|
185
|
+
embedding, embedding_dimensions, embedding_model,
|
|
186
|
+
tags, metadata, created_at, updated_at, expires_at, status
|
|
187
|
+
) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`;
|
|
188
|
+
db.run(insertSql, [
|
|
189
|
+
id,
|
|
190
|
+
key,
|
|
191
|
+
namespace,
|
|
192
|
+
value,
|
|
193
|
+
embeddingJson,
|
|
194
|
+
embeddingDimensions,
|
|
195
|
+
embeddingModel,
|
|
196
|
+
tags.length > 0 ? JSON.stringify(tags) : null,
|
|
197
|
+
serialiseMetadata(options.metadata),
|
|
198
|
+
now,
|
|
199
|
+
now,
|
|
200
|
+
ttl ? now + (ttl * 1000) : null
|
|
201
|
+
]);
|
|
202
|
+
// node:sqlite + WAL persisted that INSERT on commit — the sql.js
|
|
203
|
+
// whole-file `db.export()` + atomicWriteFileSync that lived here was
|
|
204
|
+
// the multi-writer clobber vector epic #1078 killed structurally.
|
|
205
|
+
// Query exact stats while DB is still open. `missing` is the active-rows-
|
|
206
|
+
// with-NULL-embedding count, surfaced via vector-stats.json so the
|
|
207
|
+
// statusline can warn on coverage holes (#648 / #649).
|
|
208
|
+
let vecCount = 0, nsCount = 0, missingCount = 0;
|
|
209
|
+
try {
|
|
210
|
+
const vc = db.exec("SELECT COUNT(*) FROM memory_entries WHERE status='active' AND embedding IS NOT NULL");
|
|
211
|
+
vecCount = vc[0]?.values?.[0]?.[0] ?? 0;
|
|
212
|
+
const nc = db.exec("SELECT COUNT(DISTINCT namespace) FROM memory_entries WHERE status='active'");
|
|
213
|
+
nsCount = nc[0]?.values?.[0]?.[0] ?? 0;
|
|
214
|
+
const mc = db.exec("SELECT COUNT(*) FROM memory_entries WHERE status='active' AND embedding IS NULL");
|
|
215
|
+
missingCount = mc[0]?.values?.[0]?.[0] ?? 0;
|
|
216
|
+
}
|
|
217
|
+
catch { /* table may not have status column in older DBs */ }
|
|
218
|
+
db.close();
|
|
219
|
+
// Add to HNSW index for faster future searches
|
|
220
|
+
if (embeddingJson) {
|
|
221
|
+
const embResult = JSON.parse(embeddingJson);
|
|
222
|
+
await addToHNSWIndex(id, embResult, {
|
|
223
|
+
id,
|
|
224
|
+
key,
|
|
225
|
+
namespace,
|
|
226
|
+
content: value
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
// Update statusline cache with exact counts
|
|
230
|
+
writeVectorStatsCache(dbPath, { vectorCount: vecCount, namespaces: nsCount, missing: missingCount });
|
|
231
|
+
return {
|
|
232
|
+
success: true,
|
|
233
|
+
id,
|
|
234
|
+
embedding: embeddingJson ? { dimensions: embeddingDimensions, model: embeddingModel } : undefined
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
return {
|
|
239
|
+
success: false,
|
|
240
|
+
id: '',
|
|
241
|
+
error: errorDetail(error)
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Bulk-store entries — batches writes through the bridge in a single
|
|
247
|
+
* persist-once transaction. Falls back to sequential `storeEntry()` calls
|
|
248
|
+
* (each persisting independently) when the bridge is unavailable.
|
|
249
|
+
*/
|
|
250
|
+
export async function storeEntries(items, dbPath) {
|
|
251
|
+
if (items.length === 0)
|
|
252
|
+
return [];
|
|
253
|
+
const bridge = await getBridge();
|
|
254
|
+
if (bridge && typeof bridge.bridgeStoreEntries === 'function') {
|
|
255
|
+
const bridgeResult = await bridge.bridgeStoreEntries(items, dbPath);
|
|
256
|
+
if (bridgeResult)
|
|
257
|
+
return bridgeResult;
|
|
258
|
+
}
|
|
259
|
+
// Fallback: sequential single-entry writes (each persists). Slow but correct.
|
|
260
|
+
const out = [];
|
|
261
|
+
for (const item of items) {
|
|
262
|
+
out.push(await storeEntry({ ...item, dbPath }));
|
|
263
|
+
}
|
|
264
|
+
return out;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Delete a memory entry by key and namespace
|
|
268
|
+
* Issue #980: Properly supports namespaced entries
|
|
269
|
+
*/
|
|
270
|
+
export async function deleteEntry(options) {
|
|
271
|
+
// #981 — single-writer routing for deletes. Same gates as storeEntry:
|
|
272
|
+
// not the daemon, no custom dbPath, routing not opted out. Failure paths
|
|
273
|
+
// fall through to the existing bridge / direct-write logic below.
|
|
274
|
+
if (!options.dbPath
|
|
275
|
+
&& process.env.MOFLO_IS_DAEMON !== '1'
|
|
276
|
+
&& process.env.MOFLO_DISABLE_DAEMON_ROUTING !== '1') {
|
|
277
|
+
try {
|
|
278
|
+
const routed = await tryDaemonDelete({
|
|
279
|
+
namespace: options.namespace ?? 'default',
|
|
280
|
+
key: options.key,
|
|
281
|
+
});
|
|
282
|
+
if (routed.routed && routed.ok) {
|
|
283
|
+
return {
|
|
284
|
+
success: true,
|
|
285
|
+
deleted: routed.deleted ?? true,
|
|
286
|
+
key: options.key,
|
|
287
|
+
namespace: options.namespace ?? 'default',
|
|
288
|
+
// Daemon doesn't surface remainingEntries; callers that depend on
|
|
289
|
+
// this value (the `flo memory delete` CLI) read it from a
|
|
290
|
+
// subsequent stat query, not this return shape.
|
|
291
|
+
remainingEntries: 0,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
// #1101 — daemon rejected delete args (4xx); propagate.
|
|
295
|
+
if (routed.routed && routed.ok === false) {
|
|
296
|
+
return {
|
|
297
|
+
success: false,
|
|
298
|
+
deleted: false,
|
|
299
|
+
key: options.key,
|
|
300
|
+
namespace: options.namespace ?? 'default',
|
|
301
|
+
remainingEntries: 0,
|
|
302
|
+
error: routed.error ?? 'Daemon rejected delete request',
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
catch (err) {
|
|
307
|
+
logRoutingFault(err);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
// ADR-053: Try AgentDB v3 bridge first
|
|
311
|
+
const bridge = await getBridge();
|
|
312
|
+
if (bridge) {
|
|
313
|
+
const bridgeResult = await bridge.bridgeDeleteEntry(options);
|
|
314
|
+
if (bridgeResult)
|
|
315
|
+
return bridgeResult;
|
|
316
|
+
}
|
|
317
|
+
// Fallback: direct node:sqlite write via the unified factory.
|
|
318
|
+
const { key, namespace = 'default', dbPath: customPath } = options;
|
|
319
|
+
const dbPath = customPath || memoryDbPath(process.cwd());
|
|
320
|
+
try {
|
|
321
|
+
if (!fs.existsSync(dbPath)) {
|
|
322
|
+
return {
|
|
323
|
+
success: false,
|
|
324
|
+
deleted: false,
|
|
325
|
+
key,
|
|
326
|
+
namespace,
|
|
327
|
+
remainingEntries: 0,
|
|
328
|
+
error: 'Database not found'
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
// Ensure schema has all required columns (migration for older DBs)
|
|
332
|
+
await ensureSchemaColumns(dbPath);
|
|
333
|
+
const db = openDaemonDatabase(dbPath);
|
|
334
|
+
// Check if entry exists first
|
|
335
|
+
const checkResult = db.exec(`
|
|
336
|
+
SELECT id FROM memory_entries
|
|
337
|
+
WHERE status = 'active'
|
|
338
|
+
AND key = '${key.replace(/'/g, "''")}'
|
|
339
|
+
AND namespace = '${namespace.replace(/'/g, "''")}'
|
|
340
|
+
LIMIT 1
|
|
341
|
+
`);
|
|
342
|
+
if (!checkResult[0]?.values?.[0]) {
|
|
343
|
+
// Get remaining count before closing
|
|
344
|
+
const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
|
|
345
|
+
const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
|
|
346
|
+
db.close();
|
|
347
|
+
return {
|
|
348
|
+
success: true,
|
|
349
|
+
deleted: false,
|
|
350
|
+
key,
|
|
351
|
+
namespace,
|
|
352
|
+
remainingEntries,
|
|
353
|
+
error: `Key '${key}' not found in namespace '${namespace}'`
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
// Hard-delete the entry. Soft-delete was retired in story #728: tombstones
|
|
357
|
+
// were write-only (no code ever restored from status='deleted') and bloated
|
|
358
|
+
// the DB indefinitely.
|
|
359
|
+
db.run(`DELETE FROM memory_entries WHERE key = ? AND namespace = ? AND status = 'active'`, [key, namespace]);
|
|
360
|
+
// Get remaining count
|
|
361
|
+
const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
|
|
362
|
+
const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
|
|
363
|
+
// WAL persisted the DELETE incrementally — no whole-file dump needed.
|
|
364
|
+
db.close();
|
|
365
|
+
return {
|
|
366
|
+
success: true,
|
|
367
|
+
deleted: true,
|
|
368
|
+
key,
|
|
369
|
+
namespace,
|
|
370
|
+
remainingEntries
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
catch (error) {
|
|
374
|
+
return {
|
|
375
|
+
success: false,
|
|
376
|
+
deleted: false,
|
|
377
|
+
key,
|
|
378
|
+
namespace,
|
|
379
|
+
remainingEntries: 0,
|
|
380
|
+
error: errorDetail(error)
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
//# sourceMappingURL=entries-write.js.map
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-wide HNSW vector-index singleton (150x faster vector search).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `memory-initializer.ts` (#1203 decomposition). Owns the
|
|
5
|
+
* lazy HNSW index built from the SQLite `embedding` column (or a binary
|
|
6
|
+
* sidecar), plus add/search/status/clear. Uses the pure-TS {@link HnswLite}
|
|
7
|
+
* implementation — no native dependencies.
|
|
8
|
+
*
|
|
9
|
+
* Distinct from `hnsw-index.ts` (the standalone HNSWConfig/HNSWStats
|
|
10
|
+
* implementation): this module is the singleton the memory-CRUD path wires
|
|
11
|
+
* into via `getHNSWIndex` / `addToHNSWIndex` / `searchHNSWIndex`.
|
|
12
|
+
*
|
|
13
|
+
* @module memory/hnsw-singleton
|
|
14
|
+
*/
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
import { HnswLite } from './hnsw-lite.js';
|
|
18
|
+
import { tryLoadHnswSidecar } from './hnsw-persistence.js';
|
|
19
|
+
import { parseEmbeddingJson } from './controllers/_shared.js';
|
|
20
|
+
import { memoryDbPath } from '../services/moflo-paths.js';
|
|
21
|
+
import { openDaemonDatabase } from './daemon-backend.js';
|
|
22
|
+
import { getBridge, isBridgeLoaded } from './bridge-loader.js';
|
|
23
|
+
let hnswIndex = null;
|
|
24
|
+
let hnswInitializing = false;
|
|
25
|
+
/**
|
|
26
|
+
* Get or create the HNSW index singleton
|
|
27
|
+
* Lazily initializes from SQLite data on first use
|
|
28
|
+
*/
|
|
29
|
+
export async function getHNSWIndex(options) {
|
|
30
|
+
const dimensions = options?.dimensions ?? 384;
|
|
31
|
+
// Return existing index if already initialized
|
|
32
|
+
if (hnswIndex?.initialized && !options?.forceRebuild) {
|
|
33
|
+
return hnswIndex;
|
|
34
|
+
}
|
|
35
|
+
// Prevent concurrent initialization
|
|
36
|
+
if (hnswInitializing) {
|
|
37
|
+
// Wait for initialization to complete
|
|
38
|
+
while (hnswInitializing) {
|
|
39
|
+
await new Promise(resolve => setTimeout(resolve, 10));
|
|
40
|
+
}
|
|
41
|
+
return hnswIndex;
|
|
42
|
+
}
|
|
43
|
+
hnswInitializing = true;
|
|
44
|
+
try {
|
|
45
|
+
// Use HnswLite pure TS implementation (no native dependencies).
|
|
46
|
+
// Persistent storage paths — colocated with the canonical memory DB.
|
|
47
|
+
const dbPath = options?.dbPath || memoryDbPath(process.cwd());
|
|
48
|
+
const dbDir = path.dirname(dbPath);
|
|
49
|
+
if (!fs.existsSync(dbDir)) {
|
|
50
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
51
|
+
}
|
|
52
|
+
const projectRoot = path.dirname(dbDir);
|
|
53
|
+
// Try the binary sidecar first — graph + neighbors round-trip exactly,
|
|
54
|
+
// so the cold-start cost drops to one readFileSync + slice. Fall back
|
|
55
|
+
// to SQL-rebuild only when the sidecar is missing or malformed.
|
|
56
|
+
const loadedFromSidecar = options?.forceRebuild ? null : tryLoadHnswSidecar(projectRoot);
|
|
57
|
+
const hnsw = loadedFromSidecar ?? new HnswLite(dimensions, 16, 200, 'cosine');
|
|
58
|
+
const sidecarLoaded = loadedFromSidecar !== null;
|
|
59
|
+
const db = {
|
|
60
|
+
insert: async (entry) => {
|
|
61
|
+
hnsw.add(entry.id, entry.vector);
|
|
62
|
+
},
|
|
63
|
+
search: async (query) => {
|
|
64
|
+
return hnsw.search(query.vector, query.k);
|
|
65
|
+
},
|
|
66
|
+
len: async () => hnsw.size,
|
|
67
|
+
};
|
|
68
|
+
const entries = new Map();
|
|
69
|
+
hnswIndex = {
|
|
70
|
+
db,
|
|
71
|
+
entries,
|
|
72
|
+
dimensions,
|
|
73
|
+
initialized: false
|
|
74
|
+
};
|
|
75
|
+
// Always populate the entries metadata from SQL — `key/namespace/content`
|
|
76
|
+
// is the source of truth there, and the sidecar only stores vectors +
|
|
77
|
+
// adjacency. When the sidecar IS loaded we skip the per-row JSON.parse
|
|
78
|
+
// of the embedding column, which is the expensive part on a populated
|
|
79
|
+
// consumer DB.
|
|
80
|
+
const SELECT_WITH_EMBEDDING = `id, key, namespace, content, metadata, embedding`;
|
|
81
|
+
const SELECT_METADATA_ONLY = `id, key, namespace, content, metadata`;
|
|
82
|
+
if (fs.existsSync(dbPath)) {
|
|
83
|
+
try {
|
|
84
|
+
const sqlDb = openDaemonDatabase(dbPath);
|
|
85
|
+
const cols = sidecarLoaded ? SELECT_METADATA_ONLY : SELECT_WITH_EMBEDDING;
|
|
86
|
+
const result = sqlDb.exec(`
|
|
87
|
+
SELECT ${cols}
|
|
88
|
+
FROM memory_entries
|
|
89
|
+
WHERE status = 'active' AND embedding IS NOT NULL
|
|
90
|
+
LIMIT 10000
|
|
91
|
+
`);
|
|
92
|
+
let parseSkipped = 0;
|
|
93
|
+
if (result[0]?.values) {
|
|
94
|
+
for (const row of result[0].values) {
|
|
95
|
+
// Column order matches SELECT_WITH_EMBEDDING / SELECT_METADATA_ONLY.
|
|
96
|
+
// When sidecar is loaded, embeddingJson is undefined (column absent).
|
|
97
|
+
const [id, key, ns, content, metadataJson, embeddingJson] = row;
|
|
98
|
+
if (!sidecarLoaded) {
|
|
99
|
+
const vec = parseEmbeddingJson(embeddingJson);
|
|
100
|
+
if (!vec) {
|
|
101
|
+
parseSkipped++;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
await db.insert({ id: String(id), vector: vec });
|
|
105
|
+
}
|
|
106
|
+
hnswIndex.entries.set(String(id), {
|
|
107
|
+
id: String(id),
|
|
108
|
+
key: key || String(id),
|
|
109
|
+
namespace: ns || 'default',
|
|
110
|
+
content: content || '',
|
|
111
|
+
metadata: metadataJson || undefined
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (parseSkipped > 0) {
|
|
116
|
+
console.warn(`[memory-initializer] skipped ${parseSkipped} rows with malformed embeddings`);
|
|
117
|
+
}
|
|
118
|
+
sqlDb.close();
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
console.warn(`[memory-initializer] SQL load failed, starting empty: ${err.message}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
hnswIndex.initialized = true;
|
|
125
|
+
hnswInitializing = false;
|
|
126
|
+
return hnswIndex;
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
console.warn(`[memory-initializer] getHNSWIndex failed: ${err.message}`);
|
|
130
|
+
hnswInitializing = false;
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Add entry to HNSW index. Live-adds stay in-memory until the next
|
|
136
|
+
* `memory rebuild-index` run rebuilds the binary sidecar at
|
|
137
|
+
* `.moflo/hnsw.index`. The sql.js `embedding` column is the source of
|
|
138
|
+
* truth across process boundaries.
|
|
139
|
+
*/
|
|
140
|
+
export async function addToHNSWIndex(id, embedding, entry) {
|
|
141
|
+
// ADR-053: Try AgentDB v3 bridge first
|
|
142
|
+
const bridge = await getBridge();
|
|
143
|
+
if (bridge) {
|
|
144
|
+
const bridgeResult = await bridge.bridgeAddToHNSW(id, embedding, entry);
|
|
145
|
+
if (bridgeResult === true)
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
const index = await getHNSWIndex({ dimensions: embedding.length });
|
|
149
|
+
if (!index)
|
|
150
|
+
return false;
|
|
151
|
+
try {
|
|
152
|
+
const vector = new Float32Array(embedding);
|
|
153
|
+
await index.db.insert({
|
|
154
|
+
id,
|
|
155
|
+
vector
|
|
156
|
+
});
|
|
157
|
+
index.entries.set(id, entry);
|
|
158
|
+
return true;
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Search HNSW index (150x faster than brute-force)
|
|
166
|
+
* Returns results sorted by similarity (highest first)
|
|
167
|
+
*/
|
|
168
|
+
export async function searchHNSWIndex(queryEmbedding, options) {
|
|
169
|
+
// ADR-053: Try AgentDB v3 bridge first
|
|
170
|
+
const bridge = await getBridge();
|
|
171
|
+
if (bridge) {
|
|
172
|
+
const bridgeResult = await bridge.bridgeSearchHNSW(queryEmbedding, options);
|
|
173
|
+
if (bridgeResult)
|
|
174
|
+
return bridgeResult;
|
|
175
|
+
}
|
|
176
|
+
const index = await getHNSWIndex({ dimensions: queryEmbedding.length });
|
|
177
|
+
if (!index)
|
|
178
|
+
return null;
|
|
179
|
+
try {
|
|
180
|
+
const vector = new Float32Array(queryEmbedding);
|
|
181
|
+
const k = options?.k ?? 10;
|
|
182
|
+
// HNSW search returns results with cosine distance (lower = more similar)
|
|
183
|
+
const results = await index.db.search({ vector, k: k * 2 }); // Get extra for filtering
|
|
184
|
+
const filtered = [];
|
|
185
|
+
for (const result of results) {
|
|
186
|
+
const entry = index.entries.get(result.id);
|
|
187
|
+
if (!entry)
|
|
188
|
+
continue;
|
|
189
|
+
// Filter by namespace if specified
|
|
190
|
+
if (options?.namespace && options.namespace !== 'all' && entry.namespace !== options.namespace) {
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
// Convert cosine distance to similarity score (1 - distance)
|
|
194
|
+
// Cosine distance: 0 = identical, 2 = opposite
|
|
195
|
+
const score = 1 - (result.score / 2);
|
|
196
|
+
filtered.push({
|
|
197
|
+
id: entry.id.substring(0, 12),
|
|
198
|
+
key: entry.key || entry.id.substring(0, 15),
|
|
199
|
+
content: entry.content.substring(0, 60) + (entry.content.length > 60 ? '...' : ''),
|
|
200
|
+
score,
|
|
201
|
+
namespace: entry.namespace,
|
|
202
|
+
metadata: entry.metadata
|
|
203
|
+
});
|
|
204
|
+
if (filtered.length >= k)
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
// Sort by score descending (highest similarity first)
|
|
208
|
+
filtered.sort((a, b) => b.score - a.score);
|
|
209
|
+
return filtered;
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Get HNSW index status
|
|
217
|
+
*/
|
|
218
|
+
export function getHNSWStatus() {
|
|
219
|
+
// ADR-053: If bridge was previously loaded, report availability
|
|
220
|
+
if (isBridgeLoaded()) {
|
|
221
|
+
// Bridge is loaded — HNSW-equivalent is available via AgentDB v3
|
|
222
|
+
return {
|
|
223
|
+
available: true,
|
|
224
|
+
initialized: true,
|
|
225
|
+
entryCount: hnswIndex?.entries.size ?? 0,
|
|
226
|
+
dimensions: hnswIndex?.dimensions ?? 384
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
available: hnswIndex !== null,
|
|
231
|
+
initialized: hnswIndex?.initialized ?? false,
|
|
232
|
+
entryCount: hnswIndex?.entries.size ?? 0,
|
|
233
|
+
dimensions: hnswIndex?.dimensions ?? 384
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Clear the HNSW index (for rebuilding)
|
|
238
|
+
*/
|
|
239
|
+
export function clearHNSWIndex() {
|
|
240
|
+
hnswIndex = null;
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=hnsw-singleton.js.map
|