moflo 4.12.11 → 4.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-cli-reference.md +45 -1
- package/.claude/guidance/shipped/moflo-cross-install-memory-sharing.md +7 -2
- package/.claude/guidance/shipped/moflo-skills-reference.md +2 -0
- package/.claude/skills/fl/phases.md +51 -17
- package/.claude/skills/optimize-learnings/SKILL.md +220 -0
- package/README.md +95 -1
- package/bin/lib/get-backend.mjs +150 -12
- package/bin/lib/skill-categories.mjs +1 -0
- package/bin/session-start-launcher.mjs +13 -5
- package/dist/src/cli/commands/daemon.js +5 -2
- package/dist/src/cli/commands/epic.js +5 -1
- package/dist/src/cli/commands/hive-mind.js +6 -4
- package/dist/src/cli/commands/hooks.js +8 -8
- package/dist/src/cli/commands/index.js +5 -0
- package/dist/src/cli/commands/memory-audit-learnings.js +587 -0
- package/dist/src/cli/commands/memory.js +71 -10
- package/dist/src/cli/commands/spell-schedule.js +5 -3
- package/dist/src/cli/commands/worktree.js +408 -0
- package/dist/src/cli/config/moflo-config.js +57 -0
- package/dist/src/cli/index.js +4 -2
- package/dist/src/cli/init/executor.js +1 -0
- package/dist/src/cli/mcp-tools/memory-admin-tools.js +46 -8
- package/dist/src/cli/mcp-tools/moflodb-tools.js +30 -6
- package/dist/src/cli/memory/bridge-entries.js +157 -9
- package/dist/src/cli/memory/controllers/batch-operations.js +7 -2
- package/dist/src/cli/memory/daemon-backend.js +152 -11
- package/dist/src/cli/memory/entries-read.js +47 -2
- package/dist/src/cli/memory/entries-write.js +73 -10
- package/dist/src/cli/memory/hnsw-singleton.js +112 -9
- package/dist/src/cli/memory/learnings-audit.js +420 -0
- package/dist/src/cli/memory/learnings-dead-paths.js +202 -0
- package/dist/src/cli/memory/learnings-tree.js +187 -0
- package/dist/src/cli/memory/memory-bridge.js +37 -27
- package/dist/src/cli/memory/tool-call-markup.js +218 -0
- package/dist/src/cli/parser.js +7 -3
- package/dist/src/cli/services/cherry-pick-learnings.js +9 -3
- package/dist/src/cli/services/durable-reconcile.js +161 -0
- package/dist/src/cli/services/durable-store-io.js +291 -0
- package/dist/src/cli/services/durable-sync.js +159 -24
- package/dist/src/cli/services/team-artifact-sync.js +462 -163
- package/dist/src/cli/services/worktree-provision.js +400 -0
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -2
|
@@ -190,6 +190,39 @@ function coerceMemoryBackend(raw) {
|
|
|
190
190
|
}
|
|
191
191
|
return DEFAULT_CONFIG.memory.backend;
|
|
192
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Coerce a `worktree.copy` / `worktree.link` entry to a string array (#1481).
|
|
195
|
+
* Both keys accept a bare string for the common single-entry case; anything
|
|
196
|
+
* that is neither a string nor an array of strings is dropped rather than
|
|
197
|
+
* throwing — a malformed entry must never stop a consumer's config loading.
|
|
198
|
+
*/
|
|
199
|
+
function coercePathList(raw) {
|
|
200
|
+
const list = typeof raw === 'string' ? [raw] : Array.isArray(raw) ? raw : undefined;
|
|
201
|
+
if (!list)
|
|
202
|
+
return undefined;
|
|
203
|
+
const cleaned = list
|
|
204
|
+
.filter((v) => typeof v === 'string')
|
|
205
|
+
.map(v => v.trim())
|
|
206
|
+
.filter(v => v.length > 0);
|
|
207
|
+
return cleaned.length > 0 ? cleaned : undefined;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Parse the optional `worktree:` block (#1481). Returns `undefined` when the
|
|
211
|
+
* block is absent or contains nothing usable, so "not configured" stays
|
|
212
|
+
* distinguishable from "configured empty". Unknown sub-keys are ignored.
|
|
213
|
+
*/
|
|
214
|
+
function parseWorktreeConfig(raw) {
|
|
215
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
216
|
+
return undefined;
|
|
217
|
+
const block = raw;
|
|
218
|
+
const dir = typeof block.dir === 'string' && block.dir.trim().length > 0 ? block.dir.trim() : undefined;
|
|
219
|
+
const copy = coercePathList(block.copy);
|
|
220
|
+
const link = coercePathList(block.link);
|
|
221
|
+
const setup = typeof block.setup === 'string' && block.setup.trim().length > 0 ? block.setup.trim() : undefined;
|
|
222
|
+
if (!dir && !copy && !link && !setup)
|
|
223
|
+
return undefined;
|
|
224
|
+
return { ...(dir && { dir }), ...(copy && { copy }), ...(link && { link }), ...(setup && { setup }) };
|
|
225
|
+
}
|
|
193
226
|
/**
|
|
194
227
|
* Parse raw config object into typed config, merging with defaults.
|
|
195
228
|
*/
|
|
@@ -265,6 +298,10 @@ function mergeConfig(raw, root) {
|
|
|
265
298
|
return typeof v === 'string' && v.trim().length > 0 ? v.trim() : undefined;
|
|
266
299
|
})(),
|
|
267
300
|
},
|
|
301
|
+
// #1481 — optional worktree provisioning. The whole block stays `undefined`
|
|
302
|
+
// when absent so an existing consumer `moflo.yaml` is unaffected, and so
|
|
303
|
+
// `flo worktree add` can distinguish "not configured" from "configured empty".
|
|
304
|
+
worktree: parseWorktreeConfig(raw.worktree),
|
|
268
305
|
hooks: {
|
|
269
306
|
pre_edit: raw.hooks?.pre_edit ?? raw.hooks?.preEdit ?? DEFAULT_CONFIG.hooks.pre_edit,
|
|
270
307
|
post_edit: raw.hooks?.post_edit ?? raw.hooks?.postEdit ?? DEFAULT_CONFIG.hooks.post_edit,
|
|
@@ -508,6 +545,26 @@ memory:
|
|
|
508
545
|
# worktrees never produce. Conductor recipe: set hydrate_from AND snapshot_to
|
|
509
546
|
# to the SAME absolute path. Overridable per-process via MOFLO_SNAPSHOT_TO.
|
|
510
547
|
|
|
548
|
+
# Worktree provisioning (#1481) — makes "flo worktree add" (and "/flo -wt")
|
|
549
|
+
# produce a RUNNABLE workspace, not just a valid checkout. Entirely optional:
|
|
550
|
+
# with this block absent, a new worktree is created and nothing is provisioned.
|
|
551
|
+
# worktree:
|
|
552
|
+
# dir: ../myrepo-worktrees
|
|
553
|
+
# Where worktrees are created. Defaults to <repo-parent>/<repo>-worktrees.
|
|
554
|
+
# copy: [".env", ".env.*"]
|
|
555
|
+
# Gitignored files a fresh checkout lacks, copied from the primary checkout.
|
|
556
|
+
# Sources must live inside the primary checkout. NOTE: this relocates secret
|
|
557
|
+
# material to a directory OUTSIDE the repo and outside its .gitignore — keep
|
|
558
|
+
# the worktree dir out of any repo you commit.
|
|
559
|
+
# link: ["node_modules"]
|
|
560
|
+
# Symlinked (junctioned on Windows) from the primary checkout. Opt-in with no
|
|
561
|
+
# default: a symlinked root node_modules is fragile under npm workspaces —
|
|
562
|
+
# prefer "setup: npm ci" if your project uses them.
|
|
563
|
+
# setup: "npm ci"
|
|
564
|
+
# Run inside the new worktree after copy/link, with MOFLO_WORKTREE_INDEX in
|
|
565
|
+
# its environment (a small integer unique among live worktrees) so a project
|
|
566
|
+
# with fixed dev-server ports can offset them per workspace.
|
|
567
|
+
|
|
511
568
|
# Hook toggles (all on by default — disable to slim down)
|
|
512
569
|
hooks:
|
|
513
570
|
pre_edit: true # Track file edits for learning
|
package/dist/src/cli/index.js
CHANGED
|
@@ -147,7 +147,7 @@ export class CLI {
|
|
|
147
147
|
this.showVersion();
|
|
148
148
|
return;
|
|
149
149
|
}
|
|
150
|
-
if (flags.
|
|
150
|
+
if (flags.color === false) {
|
|
151
151
|
this.output.setColorEnabled(false);
|
|
152
152
|
}
|
|
153
153
|
// Set verbosity level based on flags
|
|
@@ -165,7 +165,9 @@ export class CLI {
|
|
|
165
165
|
this.output.printDebug(`CWD: ${process.cwd()}`);
|
|
166
166
|
}
|
|
167
167
|
// Run startup update check (non-blocking, silent on skip)
|
|
168
|
-
|
|
168
|
+
// `--no-update` parses to `update = false`; `noUpdate` was never set, so
|
|
169
|
+
// the check ran even when the user asked it not to (#1474).
|
|
170
|
+
if (flags.update !== false && commandPath[0] !== 'update') {
|
|
169
171
|
this.checkForUpdatesOnStartup().catch(() => { });
|
|
170
172
|
}
|
|
171
173
|
// Auto-start daemon if configured and not already running (non-blocking).
|
|
@@ -57,6 +57,7 @@ export const SKILLS_MAP = {
|
|
|
57
57
|
'vector-search',
|
|
58
58
|
'memory-worktree', // guided memory.durable_path setup (same-machine worktrees)
|
|
59
59
|
'memory-team', // guided memory.team_artifact setup + PR pre-commit hook
|
|
60
|
+
'optimize-learnings', // curation pass over the `learnings` namespace, wrapping `flo memory audit-learnings`
|
|
60
61
|
],
|
|
61
62
|
spells: [
|
|
62
63
|
'spell-builder',
|
|
@@ -24,6 +24,7 @@ import { BACKEND_LABEL } from '../memory/database-provider.js';
|
|
|
24
24
|
import { ensureInitialized } from './memory-tools.js';
|
|
25
25
|
import { memoryDbPath } from '../services/moflo-paths.js';
|
|
26
26
|
import { resolveStateRoot } from '../services/project-root.js';
|
|
27
|
+
import { DURABLE_NAMESPACES } from '../services/cherry-pick-learnings.js';
|
|
27
28
|
function dbPath() {
|
|
28
29
|
return memoryDbPath(resolveStateRoot());
|
|
29
30
|
}
|
|
@@ -281,7 +282,7 @@ export const memoryAdminTools = [
|
|
|
281
282
|
},
|
|
282
283
|
{
|
|
283
284
|
name: 'memory_cleanup',
|
|
284
|
-
description: 'Find and optionally delete expired, stale, or unusable memory entries',
|
|
285
|
+
description: 'Find and optionally delete expired, stale, or unusable memory entries. Durable namespaces (learnings, knowledge) are exempt from the age-based buckets unless named via `namespace`; TTL-expired rows are collected everywhere.',
|
|
285
286
|
category: 'memory',
|
|
286
287
|
inputSchema: {
|
|
287
288
|
type: 'object',
|
|
@@ -290,7 +291,7 @@ export const memoryAdminTools = [
|
|
|
290
291
|
dryRun: { type: 'boolean', description: 'Ignored. Cleanup is dry unless apply:true is passed; accepted only so older callers do not error.' },
|
|
291
292
|
olderThan: { type: 'string', description: 'Age cutoff for stale/unusable entries, e.g. "30d"' },
|
|
292
293
|
expiredOnly: { type: 'boolean', description: 'Only consider TTL-expired entries' },
|
|
293
|
-
namespace: { type: 'string', description: 'Restrict cleanup to one namespace' },
|
|
294
|
+
namespace: { type: 'string', description: 'Restrict cleanup to one namespace. Naming a durable namespace (learnings, knowledge) also opts it back into the age-based buckets it is exempt from by default.' },
|
|
294
295
|
},
|
|
295
296
|
},
|
|
296
297
|
handler: async (input) => {
|
|
@@ -308,10 +309,28 @@ export const memoryAdminTools = [
|
|
|
308
309
|
const now = Date.now();
|
|
309
310
|
const nsClause = namespace ? ` AND namespace = ${sqlString(namespace)}` : '';
|
|
310
311
|
const select = (cond) => `SELECT key, namespace FROM memory_entries WHERE status = 'active' AND ${cond}${nsClause}`;
|
|
312
|
+
// #1464 — durable namespaces are exempt from the two AGE-based buckets
|
|
313
|
+
// unless the caller names one explicitly.
|
|
314
|
+
//
|
|
315
|
+
// Age is not evidence of worthlessness for a learning: a two-year-old
|
|
316
|
+
// architectural decision is routinely the most valuable row in the store.
|
|
317
|
+
// Worse, `COALESCE(last_accessed_at, updated_at, created_at)` collapses to
|
|
318
|
+
// `created_at` for any row nothing has ever bumped — and until #1464 the
|
|
319
|
+
// search path, which is how learnings are actually read, bumped nothing.
|
|
320
|
+
// So "stale (unused)" silently meant "old", and the only purge surface
|
|
321
|
+
// moflo ships hit the most-consulted learnings exactly as hard as the dead
|
|
322
|
+
// ones.
|
|
323
|
+
//
|
|
324
|
+
// A DEFAULT, not a prohibition — `--namespace learnings` still collects
|
|
325
|
+
// them. TTL-expired rows stay in scope in every namespace; durable rows
|
|
326
|
+
// never set a TTL, so nothing durable is lost through that bucket.
|
|
327
|
+
const exemptDurable = !namespace;
|
|
328
|
+
const durableIn = DURABLE_NAMESPACES.map(sqlString).join(', ');
|
|
329
|
+
const durableClause = exemptDurable ? ` AND namespace NOT IN (${durableIn})` : '';
|
|
311
330
|
const expired = await query(select(`expires_at IS NOT NULL AND expires_at < ${now}`));
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
:
|
|
331
|
+
const ageCutoff = staleMs != null ? now - staleMs : null;
|
|
332
|
+
const staleCond = ageCutoff == null ? null
|
|
333
|
+
: `expires_at IS NULL AND COALESCE(last_accessed_at, updated_at, created_at) < ${ageCutoff}`;
|
|
315
334
|
// "Unusable" = no embedding (so invisible to semantic search), never
|
|
316
335
|
// read back, AND older than the caller's cutoff.
|
|
317
336
|
//
|
|
@@ -321,10 +340,27 @@ export const memoryAdminTools = [
|
|
|
321
340
|
// selected the entire store for deletion by default. Requiring an
|
|
322
341
|
// explicit cutoff means an unqualified cleanup can only ever remove
|
|
323
342
|
// TTL-expired rows.
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
`AND COALESCE(last_accessed_at, updated_at, created_at) < ${
|
|
343
|
+
const lowQualityCond = ageCutoff == null ? null
|
|
344
|
+
: `embedding IS NULL AND COALESCE(access_count, 0) = 0 `
|
|
345
|
+
+ `AND COALESCE(last_accessed_at, updated_at, created_at) < ${ageCutoff}`;
|
|
346
|
+
const ageBuckets = !expiredOnly && staleCond != null && lowQualityCond != null;
|
|
347
|
+
const stale = ageBuckets ? await query(select(staleCond + durableClause)) : [];
|
|
348
|
+
const lowQuality = ageBuckets ? await query(select(lowQualityCond + durableClause)) : [];
|
|
349
|
+
// Count what the exemption withheld. Without this the operator reads a
|
|
350
|
+
// clean result as "learnings are already tidy" rather than "learnings were
|
|
351
|
+
// not examined" — the same class of quiet lie the missing usage signal was.
|
|
352
|
+
//
|
|
353
|
+
// The TTL exclusion is not cosmetic: `lowQualityCond` does not test
|
|
354
|
+
// `expires_at`, so without it a durable row with an elapsed TTL would be
|
|
355
|
+
// reported as held back in the same call that deletes it through the
|
|
356
|
+
// expired bucket.
|
|
357
|
+
const heldBackRows = ageBuckets && exemptDurable
|
|
358
|
+
? await query(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`
|
|
359
|
+
+ ` AND namespace IN (${durableIn})`
|
|
360
|
+
+ ` AND NOT (expires_at IS NOT NULL AND expires_at < ${now})`
|
|
361
|
+
+ ` AND ((${staleCond}) OR (${lowQualityCond}))`)
|
|
327
362
|
: [];
|
|
363
|
+
const durableHeldBack = heldBackRows.length ? Number(heldBackRows[0][0] ?? 0) : 0;
|
|
328
364
|
const seen = new Set();
|
|
329
365
|
const targets = [];
|
|
330
366
|
for (const rows of [expired, stale, lowQuality]) {
|
|
@@ -348,6 +384,7 @@ export const memoryAdminTools = [
|
|
|
348
384
|
return {
|
|
349
385
|
dryRun: true,
|
|
350
386
|
candidates,
|
|
387
|
+
durableHeldBack,
|
|
351
388
|
deleted: { entries: 0 },
|
|
352
389
|
freed: { bytes: 0, formatted: '0 B' },
|
|
353
390
|
duration: Date.now() - started,
|
|
@@ -365,6 +402,7 @@ export const memoryAdminTools = [
|
|
|
365
402
|
return {
|
|
366
403
|
dryRun: false,
|
|
367
404
|
candidates,
|
|
405
|
+
durableHeldBack,
|
|
368
406
|
deleted: { entries: deleted },
|
|
369
407
|
freed: { bytes: freedBytes, formatted: formatBytes(freedBytes) },
|
|
370
408
|
duration: Date.now() - started,
|
|
@@ -433,17 +433,38 @@ export const moflodbConsolidate = {
|
|
|
433
433
|
}
|
|
434
434
|
},
|
|
435
435
|
};
|
|
436
|
-
// ===== moflodb_batch — Batch
|
|
436
|
+
// ===== moflodb_batch — Batch insert into the episodes store =====
|
|
437
|
+
/**
|
|
438
|
+
* `update` and `delete` were removed in #1465 — both reported `success: true`
|
|
439
|
+
* with a count taken from the input array while changing nothing the caller
|
|
440
|
+
* could address. Rejected here, at the tool boundary, naming the tool that
|
|
441
|
+
* does the job.
|
|
442
|
+
*
|
|
443
|
+
* A Map, not an object literal: `operation` is caller-controlled, and a plain
|
|
444
|
+
* object would resolve 'constructor'/'toString'/'__proto__' up the prototype
|
|
445
|
+
* chain to a truthy function, returning an error whose message JSON-serializes
|
|
446
|
+
* to nothing.
|
|
447
|
+
*/
|
|
448
|
+
const REMOVED_BATCH_OPERATIONS = new Map([
|
|
449
|
+
[
|
|
450
|
+
'delete',
|
|
451
|
+
"moflodb_batch no longer supports 'delete' (#1465): it targeted the episodes store and could not address a namespaced memory entry, while reporting success. Use memory_delete with an explicit namespace.",
|
|
452
|
+
],
|
|
453
|
+
[
|
|
454
|
+
'update',
|
|
455
|
+
"moflodb_batch no longer supports 'update' (#1465): it targeted the episodes store and could not address a namespaced memory entry, while reporting success. Use memory_store to overwrite an entry.",
|
|
456
|
+
],
|
|
457
|
+
]);
|
|
437
458
|
export const moflodbBatch = {
|
|
438
459
|
name: 'moflodb_batch',
|
|
439
|
-
description: 'Batch
|
|
460
|
+
description: 'Batch-insert episodes into the MofloDb bridge store. Use memory_delete to remove entries and memory_store to overwrite them.',
|
|
440
461
|
inputSchema: {
|
|
441
462
|
type: 'object',
|
|
442
463
|
properties: {
|
|
443
464
|
operation: {
|
|
444
465
|
type: 'string',
|
|
445
|
-
description:
|
|
446
|
-
enum: ['insert'
|
|
466
|
+
description: "Batch operation type. Only 'insert' is supported; 'update'/'delete' were removed in #1465.",
|
|
467
|
+
enum: ['insert'],
|
|
447
468
|
},
|
|
448
469
|
entries: {
|
|
449
470
|
type: 'array',
|
|
@@ -465,8 +486,11 @@ export const moflodbBatch = {
|
|
|
465
486
|
const operation = validateString(params.operation, 'operation', 20);
|
|
466
487
|
if (!operation)
|
|
467
488
|
return { success: false, error: 'operation is required (string)' };
|
|
468
|
-
|
|
469
|
-
|
|
489
|
+
const removed = REMOVED_BATCH_OPERATIONS.get(operation);
|
|
490
|
+
if (removed)
|
|
491
|
+
return { success: false, error: removed };
|
|
492
|
+
if (operation !== 'insert') {
|
|
493
|
+
return { success: false, error: `Invalid operation: ${operation}. Must be insert` };
|
|
470
494
|
}
|
|
471
495
|
if (!Array.isArray(params.entries) || params.entries.length === 0) {
|
|
472
496
|
return { success: false, error: 'entries is required (non-empty array)' };
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { cosineSim, execRows, generateId, logBridgeError, persistBridgeDb, refreshVectorStatsCache, searchCandidateCap, withDb } from './bridge-core.js';
|
|
11
11
|
import { embeddingResponseFrom, getBridgeEmbedder, resolveBridgeEmbedding } from './bridge-embedder.js';
|
|
12
12
|
import { errorDetail } from '../shared/utils/error-detail.js';
|
|
13
|
+
import { archiveDurableRow, isDurableNamespace } from '../services/durable-store-io.js';
|
|
13
14
|
/**
|
|
14
15
|
* Run `persistBridgeDb` and convert any throw into a `persist failed:`
|
|
15
16
|
* error string for the caller. Centralises the #982 single-store /
|
|
@@ -59,6 +60,112 @@ function makeEntryCacheKey(namespace, key) {
|
|
|
59
60
|
* trade; a systematic undercount of HOT keys — the #1396 defect — is not.
|
|
60
61
|
*/
|
|
61
62
|
const ACCESS_FLUSH_INTERVAL_MS = 30_000;
|
|
63
|
+
/**
|
|
64
|
+
* The access bump, shared by the two throttles that issue it. Adding the
|
|
65
|
+
* accumulated delta in SQL — rather than writing a client-computed absolute —
|
|
66
|
+
* is what keeps the counter correct under concurrency, so the statement is
|
|
67
|
+
* written once and reused rather than retyped per call site.
|
|
68
|
+
*/
|
|
69
|
+
const ACCESS_BUMP_SQL = `UPDATE memory_entries SET access_count = access_count + ?, last_accessed_at = ? WHERE id = ?`;
|
|
70
|
+
/**
|
|
71
|
+
* Deferred `access_count` deltas for rows returned by SEARCH, per database.
|
|
72
|
+
*
|
|
73
|
+
* #1464 — `memory_search` is the read path for durable learnings (CLAUDE.md
|
|
74
|
+
* routes every prompt through it before any other read), but nothing on that
|
|
75
|
+
* path recorded usage: `access_count` / `last_accessed_at` moved only on
|
|
76
|
+
* retrieve-by-key. The most-consulted learning in the store looked untouched
|
|
77
|
+
* since the day it was written, which in turn made every age-based cleanup
|
|
78
|
+
* heuristic a guess dressed up as a measurement.
|
|
79
|
+
*
|
|
80
|
+
* Scoped to DURABLE namespaces on purpose. Structural namespaces (code-map,
|
|
81
|
+
* patterns, tests) are re-indexed wholesale on a schedule and their usage
|
|
82
|
+
* counts are noise — paying a write for them would tax the hot path to record
|
|
83
|
+
* nothing anyone reads.
|
|
84
|
+
*
|
|
85
|
+
* KEYED BY THE DATABASE HANDLE, not module-global. Entry ids are only
|
|
86
|
+
* meaningful inside the store that issued them, so a process that reaches a
|
|
87
|
+
* second database — a `dbPath` override, or a bridge rebuilt against a
|
|
88
|
+
* different project root — must not carry the first one's deltas across.
|
|
89
|
+
* Module-global state would flush ids that match nothing in the new store and
|
|
90
|
+
* then clear them, silently discarding counts against the "defer, never lose"
|
|
91
|
+
* rule below. A WeakMap also means a torn-down bridge's state is collected with
|
|
92
|
+
* its handle rather than accumulating for the life of the daemon, and each test
|
|
93
|
+
* gets clean state from its own database with no reset hook to remember.
|
|
94
|
+
*
|
|
95
|
+
* Same trade as the per-key entry-cache throttle below: defer writes, never
|
|
96
|
+
* lose counts. The flush stamp is per DATABASE rather than per key because a
|
|
97
|
+
* search touches a whole result set at once — the unit being coalesced here is
|
|
98
|
+
* the search, not the key.
|
|
99
|
+
*/
|
|
100
|
+
const searchAccessByDb = new WeakMap();
|
|
101
|
+
/**
|
|
102
|
+
* Hard bound on one database's pending deltas. Durable namespaces are small, so
|
|
103
|
+
* this is a backstop rather than a working limit — but an unbounded map inside
|
|
104
|
+
* a daemon that lives for days is a leak regardless of how unlikely it is to
|
|
105
|
+
* fill. Reaching the cap forces a flush; it never drops deltas.
|
|
106
|
+
*/
|
|
107
|
+
const SEARCH_ACCESS_PENDING_CAP = 1_000;
|
|
108
|
+
/**
|
|
109
|
+
* Accumulate one access per returned durable row, flushing at most once per
|
|
110
|
+
* {@link ACCESS_FLUSH_INTERVAL_MS}.
|
|
111
|
+
*
|
|
112
|
+
* `ids` are already filtered to durable rows by the caller, which is also where
|
|
113
|
+
* the per-row namespace test happens — a structural hit never reaches this map.
|
|
114
|
+
* Call with an empty array to give a pending set its chance to flush.
|
|
115
|
+
*
|
|
116
|
+
* Best-effort by construction: a throw leaves the deltas pending for the next
|
|
117
|
+
* attempt and search results are returned either way. Usage is observability,
|
|
118
|
+
* not correctness — #1058 is the standing proof of what happens when the read
|
|
119
|
+
* path takes on a write obligation it cannot honour safely. What it issues is a
|
|
120
|
+
* bounded per-row UPDATE, never the whole-DB `db.export()` writeback that
|
|
121
|
+
* clobbered concurrent writers.
|
|
122
|
+
*/
|
|
123
|
+
function recordSearchAccess(db, ids, now) {
|
|
124
|
+
let state = searchAccessByDb.get(db);
|
|
125
|
+
if (!state) {
|
|
126
|
+
// Nothing to record and nothing pending — don't allocate state for a
|
|
127
|
+
// database whose searches never return a durable row.
|
|
128
|
+
if (ids.length === 0)
|
|
129
|
+
return;
|
|
130
|
+
state = { deltas: new Map(), lastFlushAt: 0 };
|
|
131
|
+
searchAccessByDb.set(db, state);
|
|
132
|
+
}
|
|
133
|
+
for (const id of ids)
|
|
134
|
+
state.deltas.set(id, (state.deltas.get(id) ?? 0) + 1);
|
|
135
|
+
if (state.deltas.size === 0)
|
|
136
|
+
return;
|
|
137
|
+
// A database this process has never flushed writes immediately rather than
|
|
138
|
+
// waiting out the interval — moflo's CLI processes are short-lived and would
|
|
139
|
+
// otherwise exit with every access still pending, reintroducing the silent
|
|
140
|
+
// undercount this exists to remove.
|
|
141
|
+
const forced = state.deltas.size >= SEARCH_ACCESS_PENDING_CAP;
|
|
142
|
+
if (!forced && now - state.lastFlushAt < ACCESS_FLUSH_INTERVAL_MS)
|
|
143
|
+
return;
|
|
144
|
+
try {
|
|
145
|
+
const stmt = db.prepare(ACCESS_BUMP_SQL);
|
|
146
|
+
db.run('BEGIN');
|
|
147
|
+
try {
|
|
148
|
+
for (const [id, delta] of state.deltas)
|
|
149
|
+
stmt.run([delta, now, id]);
|
|
150
|
+
db.run('COMMIT');
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
try {
|
|
154
|
+
db.run('ROLLBACK');
|
|
155
|
+
}
|
|
156
|
+
catch { /* a failed COMMIT already ended the txn */ }
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
// Clear ONLY after the commit lands. Clearing on a throw would discard the
|
|
160
|
+
// accumulated hits outright — the throttle defers writes, it does not drop
|
|
161
|
+
// them.
|
|
162
|
+
state.deltas.clear();
|
|
163
|
+
state.lastFlushAt = now;
|
|
164
|
+
}
|
|
165
|
+
catch (err) {
|
|
166
|
+
logBridgeError('search access flush failed', err);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
62
169
|
/** Normalise `metadata` for the `metadata` TEXT column; `undefined` → `'{}'` (#1064). */
|
|
63
170
|
export function serialiseMetadata(metadata) {
|
|
64
171
|
if (metadata == null)
|
|
@@ -544,6 +651,15 @@ export async function bridgeSearchEntries(options) {
|
|
|
544
651
|
const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, rows);
|
|
545
652
|
const docCount = rows.length;
|
|
546
653
|
const results = [];
|
|
654
|
+
// #1464 — usage recording needs the FULL row id; the emitted `id` above is
|
|
655
|
+
// truncated to 12 chars for the envelope and would match no row. Keyed by
|
|
656
|
+
// the result object rather than by index because `results` is sorted and
|
|
657
|
+
// sliced before the returned set is known. Never spread into the response.
|
|
658
|
+
//
|
|
659
|
+
// Null for a namespace-scoped search of a structural namespace: nothing it
|
|
660
|
+
// returns can be durable, so it skips the bookkeeping outright rather than
|
|
661
|
+
// testing every row against a set that will never match.
|
|
662
|
+
const durableIdByResult = namespace === 'all' || isDurableNamespace(namespace) ? new Map() : null;
|
|
547
663
|
for (const row of rows) {
|
|
548
664
|
let semanticScore = 0;
|
|
549
665
|
let bm25ScoreVal = 0;
|
|
@@ -568,7 +684,7 @@ export async function bridgeSearchEntries(options) {
|
|
|
568
684
|
? `semantic:${semanticScore.toFixed(3)}+bm25:${bm25ScoreVal.toFixed(3)}`
|
|
569
685
|
: `bm25:${bm25ScoreVal.toFixed(3)}`;
|
|
570
686
|
const metadataStr = row.metadata != null ? String(row.metadata) : undefined;
|
|
571
|
-
|
|
687
|
+
const hit = {
|
|
572
688
|
id: String(row.id).substring(0, 12),
|
|
573
689
|
// The substring is a fallback id-prefix when key is missing —
|
|
574
690
|
// applying it to the full expression truncates valid keys (#845).
|
|
@@ -578,13 +694,34 @@ export async function bridgeSearchEntries(options) {
|
|
|
578
694
|
namespace: String(row.namespace || 'default'),
|
|
579
695
|
provenance,
|
|
580
696
|
metadata: metadataStr,
|
|
581
|
-
}
|
|
697
|
+
};
|
|
698
|
+
results.push(hit);
|
|
699
|
+
// Per row, not per search: an `all`-namespace search returns mostly
|
|
700
|
+
// structural hits, and storing an id only to discard it at flush time
|
|
701
|
+
// is work every prompt in every consumer project would pay.
|
|
702
|
+
if (durableIdByResult && isDurableNamespace(hit.namespace)) {
|
|
703
|
+
durableIdByResult.set(hit, String(row.id));
|
|
704
|
+
}
|
|
582
705
|
}
|
|
583
706
|
}
|
|
584
707
|
results.sort((a, b) => b.score - a.score);
|
|
708
|
+
const returned = results.slice(0, limit);
|
|
709
|
+
// #1464 — record usage for the durable rows this search actually returned.
|
|
710
|
+
// Placed after the slice so an over-fetched candidate the caller never sees
|
|
711
|
+
// does not count as a read. Called even when this search returned none, so
|
|
712
|
+
// a set left pending by an earlier search still gets its flush.
|
|
713
|
+
if (durableIdByResult) {
|
|
714
|
+
const durableIds = [];
|
|
715
|
+
for (const r of returned) {
|
|
716
|
+
const id = durableIdByResult.get(r);
|
|
717
|
+
if (id)
|
|
718
|
+
durableIds.push(id);
|
|
719
|
+
}
|
|
720
|
+
recordSearchAccess(ctx.db, durableIds, Date.now());
|
|
721
|
+
}
|
|
585
722
|
return {
|
|
586
723
|
success: true,
|
|
587
|
-
results:
|
|
724
|
+
results: returned,
|
|
588
725
|
searchTime: Date.now() - startTime,
|
|
589
726
|
searchMethod: queryEmbedding ? 'hybrid-bm25-semantic' : 'bm25-only',
|
|
590
727
|
};
|
|
@@ -700,7 +837,8 @@ export async function bridgeGetEntry(options) {
|
|
|
700
837
|
const lastFlushAt = cached.lastAccessFlushAt ?? 0;
|
|
701
838
|
if (now - lastFlushAt >= ACCESS_FLUSH_INTERVAL_MS) {
|
|
702
839
|
try {
|
|
703
|
-
ctx.db.prepare(
|
|
840
|
+
ctx.db.prepare(ACCESS_BUMP_SQL)
|
|
841
|
+
.run([cached.pendingAccessDelta, now, String(cached.id || '')]);
|
|
704
842
|
// Clear ONLY after the write lands. Clearing on a throw would discard
|
|
705
843
|
// the accumulated hits outright — the throttle defers writes, it does
|
|
706
844
|
// not drop them.
|
|
@@ -819,12 +957,22 @@ export async function bridgeDeleteEntry(options) {
|
|
|
819
957
|
}
|
|
820
958
|
let changes = 0;
|
|
821
959
|
try {
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
960
|
+
// Durable namespaces archive rather than hard-delete, so the deletion can
|
|
961
|
+
// reach the team artifact and sibling worktrees (#1463). Same rule as the
|
|
962
|
+
// offline path in `entries-write.deleteEntry` — see the rationale there.
|
|
963
|
+
if (isDurableNamespace(namespace)) {
|
|
964
|
+
archiveDurableRow(ctx.db, namespace, key, Date.now());
|
|
965
|
+
}
|
|
966
|
+
else {
|
|
967
|
+
ctx.db.prepare(`
|
|
968
|
+
DELETE FROM memory_entries
|
|
969
|
+
WHERE key = ? AND namespace = ? AND status = 'active'
|
|
970
|
+
`).run([key, namespace]);
|
|
971
|
+
}
|
|
826
972
|
// sql.js Statement.run returns true/false, not { changes }. Use
|
|
827
|
-
// db.getRowsModified() to read the row count from the last statement
|
|
973
|
+
// db.getRowsModified() to read the row count from the last statement —
|
|
974
|
+
// an UPDATE reports rows affected the same way, so the zero-rows
|
|
975
|
+
// inconsistency check below covers the archive path too.
|
|
828
976
|
changes = ctx.db.getRowsModified?.() ?? 0;
|
|
829
977
|
}
|
|
830
978
|
catch (err) {
|
|
@@ -6,8 +6,13 @@
|
|
|
6
6
|
*
|
|
7
7
|
* Consumer surface (from src/cli/memory/memory-bridge.ts):
|
|
8
8
|
* - insertEpisodes([{content, metadata?, embedding?}])
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
*
|
|
10
|
+
* bulkDelete/bulkUpdate remain here — they report rows actually affected and
|
|
11
|
+
* are covered by this controller's tests — but #1465 removed their only
|
|
12
|
+
* caller. `moflodb_batch` routed them at the `episodes` store through a schema
|
|
13
|
+
* with no `namespace`, so they could not address a caller's `memory_entries`
|
|
14
|
+
* row; entry deletion belongs to `memory_delete`. Do not re-wire a bridge
|
|
15
|
+
* operation to them without a namespace-aware target.
|
|
11
16
|
*
|
|
12
17
|
* Only the `episodes` table is whitelisted for delete/update to keep the
|
|
13
18
|
* SQL surface narrow; attempts to target any other table throw.
|