moflo 4.9.35 → 4.9.37

package/dist/src/cli/commands/doctor-checks-memory-access.js

@@ -22,7 +22,10 @@
  * Cleanup runs even on assertion failure so a fail doesn't leave orphaned
  * agents/hive workers.
  */
+import { existsSync, readFileSync, writeFileSync } from 'fs';
 import { errorDetail } from '../shared/utils/error-detail.js';
+import { mofloImport } from '../services/moflo-require.js';
+import { memoryDbPath } from '../services/moflo-paths.js';
 import { loadToolArrays, getTool, pushDetail, summarizeFunctional, } from './doctor-checks-functional-shared.js';
 const MEMORY_ACCESS_CHECK = 'Memory Access Functional';
 const MEMORY_ACCESS_FAIL_FIX = 'Run `flo doctor --json` for per-subcheck details. Common fixes: ensure fastembed installed (memory_store.hasEmbedding=false), explicit threshold:0 honored (#837), or rebuild HNSW index (`flo memory rebuild-index`)';
@@ -160,6 +163,164 @@ async function safeDelete(memoryTools, key, namespace) {
     }
     catch { /* best-effort */ }
 }
+/**
+ * #1053 S2: probe `memory_get_neighbors` round-trip.
+ *
+ * Same #798 protected-functionality posture as the swarm/agent/task probes:
+ * if a future refactor stubs the handler to literals (or unwires it from
+ * the metadata-passthrough plumbing in S1), this probe fails BEFORE the
+ * stub ships to consumers.
+ *
+ * Three chunks are stored via memory_store, then their `metadata` columns
+ * are written directly via sql.js to inject chunk-shaped nav fields
+ * (memory_store always sets metadata to '{}', so the chunker write path is
+ * the only producer in steady state — bypass it here for an in-process
+ * probe). The middle chunk's neighbors are then fetched and verified to
+ * carry navigation back, proving S1 + S2 + the metadata column passthrough
+ * survive end-to-end.
+ */
+async function probeMemoryGetNeighbors(memoryTools, details) {
+    const tool = getTool(memoryTools, 'memory_get_neighbors');
+    if (!tool?.handler) {
+        details.push({
+            id: 'neighbors.registered',
+            mcpTool: 'memory_get_neighbors',
+            status: 'fail',
+            observed: { registered: false },
+            expected: 'memory_get_neighbors registered in MCP tool surface (#1053 S2)',
+            message: 'memory_get_neighbors is not registered — has the tool been removed or its name changed?',
+        });
+        return null;
+    }
+    details.push({
+        id: 'neighbors.registered',
+        mcpTool: 'memory_get_neighbors',
+        status: 'pass',
+        observed: { registered: true },
+        expected: 'memory_get_neighbors registered',
+    });
+    const stamp = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const namespace = `doctor-neighbors-${stamp}`;
+    const prefix = `chunk-doctor-neighbors-${stamp}`;
+    const chunkKeys = [`${prefix}-0`, `${prefix}-1`, `${prefix}-2`];
+    const middleKey = chunkKeys[1];
+    // Seed three chunks via DIRECT sql.js write (skipping memory_store).
+    // Going through memory_store would warm the bridge's TieredCache with
+    // metadata='{}', and a follow-up direct UPDATE would race the bridge's
+    // writeback (sql.js dump-on-flush hazard, see
+    // feedback_sqljs_writeback_clobber.md). Writing straight to disk before
+    // the bridge ever sees these keys means memory_get_neighbors → getEntry
+    // hits fresh disk state and our injected metadata is what comes back.
+    const dbPath = memoryDbPath(process.cwd());
+    if (!existsSync(dbPath)) {
+        details.push({
+            id: 'neighbors.seed',
+            mcpTool: 'memory_get_neighbors',
+            status: 'warn',
+            observed: { dbPath, exists: false },
+            expected: 'memory.db present so the neighbors probe can seed chunks',
+            message: 'memory.db missing — skip the neighbors probe (run `flo memory init` first)',
+        });
+        return { key: middleKey, namespace, chunkKeys };
+    }
+    try {
+        const initSqlJs = (await mofloImport('sql.js')).default;
+        const SQL = await initSqlJs();
+        const db = new SQL.Database(readFileSync(dbPath));
+        const insert = db.prepare(`INSERT OR REPLACE INTO memory_entries
+        (id, key, namespace, content, type, tags, metadata, created_at, updated_at, status)
+       VALUES (?, ?, ?, ?, 'semantic', '[]', ?, ?, ?, 'active')`);
+        const now = Date.now();
+        for (let i = 0; i < chunkKeys.length; i++) {
+            const meta = {
+                type: 'chunk',
+                parentDoc: 'doc-doctor-neighbors',
+                parentPath: '/doctor-neighbors.md',
+                chunkIndex: i,
+                totalChunks: chunkKeys.length,
+                prevChunk: i > 0 ? chunkKeys[i - 1] : null,
+                nextChunk: i < chunkKeys.length - 1 ? chunkKeys[i + 1] : null,
+                siblings: chunkKeys,
+                hierarchicalParent: null,
+                hierarchicalChildren: null,
+                chunkTitle: `Doctor Probe Chunk ${i}`,
+                headerLevel: 2,
+                docContentHash: stamp,
+            };
+            insert.run([
+                `memprobe-neighbors-${stamp}-${i}`,
+                chunkKeys[i],
+                namespace,
+                `chunk body ${i}`,
+                JSON.stringify(meta),
+                now, now,
+            ]);
+        }
+        insert.free();
+        writeFileSync(dbPath, Buffer.from(db.export()));
+        db.close();
+    }
+    catch (err) {
+        const msg = errorDetail(err, { firstLineOnly: true });
+        details.push({
+            id: 'neighbors.seed',
+            mcpTool: 'memory_get_neighbors',
+            status: 'fail',
+            observed: { error: msg },
+            expected: 'three chunk rows seeded with chunk-shaped metadata',
+            message: `seed failed: ${msg}`,
+        });
+        return { key: middleKey, namespace, chunkKeys };
+    }
+    // 3. The probe itself — fetch prev + next of the middle chunk.
+    let result;
+    try {
+        result = (await invokeOrThrow(memoryTools, 'memory_get_neighbors', {
+            key: middleKey,
+            namespace,
+        }));
+    }
+    catch (err) {
+        const msg = errorDetail(err, { firstLineOnly: true });
+        details.push({
+            id: 'neighbors.roundtrip',
+            mcpTool: 'memory_get_neighbors',
+            status: 'fail',
+            observed: { error: msg },
+            expected: 'memory_get_neighbors returns success=true with prev + next',
+            message: `handler threw: ${msg}`,
+        });
+        return { key: middleKey, namespace, chunkKeys };
+    }
+    const failReason = assertNeighbors(result, [chunkKeys[0], chunkKeys[2]]);
+    pushDetail(details, {
+        id: 'neighbors.roundtrip',
+        mcpTool: 'memory_get_neighbors',
+        expected: `success=true, total=2, neighbors include ${chunkKeys[0]} + ${chunkKeys[2]} with navigation`,
+    }, failReason ? result : { total: result.total, neighborKeys: result.neighbors?.map(n => n.key) }, failReason);
+    return { key: middleKey, namespace, chunkKeys };
+}
+function assertNeighbors(result, expectedKeys) {
+    if (!result?.success) {
+        return `success=${result?.success} (error: ${result?.error ?? 'unknown'}) — handler did not return success`;
+    }
+    if (result.total !== expectedKeys.length) {
+        return `expected total=${expectedKeys.length}, got ${result.total} — neighbors traversal returned wrong count`;
+    }
+    const got = (result.neighbors ?? []).map(n => n.key).sort();
+    const want = [...expectedKeys].sort();
+    if (JSON.stringify(got) !== JSON.stringify(want)) {
+        return `expected neighbor keys ${JSON.stringify(want)}, got ${JSON.stringify(got)} — wrong neighbors returned`;
+    }
+    // Every neighbor must carry navigation (S1 metadata passthrough). A stub
+    // that returns shaped envelopes but null nav would pass the count check;
+    // this catches it.
+    const missingNav = (result.neighbors ?? []).filter(n => !n.navigation);
+    if (missingNav.length > 0) {
+        return `${missingNav.length} neighbor(s) returned with navigation=null — S1 metadata passthrough may be broken`;
+    }
+    return null;
+}
 export async function checkMemoryAccessFunctional() {
     const details = [];
     const mods = await loadToolArrays({
@@ -182,6 +343,24 @@ export async function checkMemoryAccessFunctional() {
     let spawnedAgentId;
     let hiveInitialized = false;
     try {
+        // ── Probe 0: memory_get_neighbors (#1053 S2) ──────────────────────────
+        // MUST RUN FIRST — before any memory_store call warms the bridge's
+        // in-memory sql.js snapshot. The probe injects chunk metadata via a
+        // direct file write to .moflo/moflo.db; once the bridge has loaded the
+        // DB into memory and done its first persist, external file writes are
+        // clobbered (sql.js dump-on-flush hazard, see
+        // feedback_sqljs_writeback_clobber.md). Running this probe first means
+        // the bridge instantiates AFTER our seed lands — its initial disk read
+        // sees our rows.
+        {
+            const probeResult = await probeMemoryGetNeighbors(memoryTools, details);
+            if (probeResult) {
+                const { namespace, chunkKeys } = probeResult;
+                for (const key of chunkKeys) {
+                    cleanups.push(() => safeDelete(memoryTools, key, namespace));
+                }
+            }
+        }
         // ── Probe 1: subagent context ─────────────────────────────────────────
         // The "subagent" path is what Claude's Task tool ends up calling: direct
         // MCP tools with no surrounding coordinator state. Failures here indicate

package/dist/src/cli/commands/memory.js

@@ -242,7 +242,7 @@ const searchCommand = {
             name: 'threshold',
             description: 'Similarity threshold (0-1)',
             type: 'number',
-            default: 0.7
+            default: 0.5
         },
         {
             name: 'type',
@@ -268,7 +268,8 @@ const searchCommand = {
         const query = ctx.flags.query || ctx.args[0];
         const namespace = ctx.flags.namespace || 'all';
         const limit = ctx.flags.limit || 10;
-        const threshold = ctx.flags.threshold || 0.3;
+        // #1053 S6: align with MCP default — was 0.3 here vs 0.7 in option block.
+        const threshold = ctx.flags.threshold || 0.5;
         const searchType = ctx.flags.type || 'semantic';
         const buildHnsw = ctx.flags.buildHnsw;
         if (!query) {
@@ -1612,16 +1613,16 @@ const indexGuidanceCommand = {
             try {
                 const content = fs.readFileSync(filePath, 'utf-8');
                 const contentHash_ = hashContent(content);
-                // Check if content changed
+                // #1053 S4: doc-* retired — read docContentHash off chunk-0 instead.
                 if (!forceReindex) {
                     const stmt = db.prepare('SELECT metadata FROM memory_entries WHERE key = ? AND namespace = ?');
-                    stmt.bind([docKey, NAMESPACE]);
+                    stmt.bind([`${chunkPrefix}-0`, NAMESPACE]);
                     const entry = stmt.step() ? stmt.getAsObject() : null;
                     stmt.free();
                     if (entry?.metadata) {
                         try {
                             const meta = JSON.parse(entry.metadata);
-                            if (meta.contentHash === contentHash_) {
+                            if (meta.docContentHash === contentHash_) {
                                 return { docKey, status: 'unchanged', chunks: 0 };
                             }
                         }
@@ -1630,19 +1631,12 @@ const indexGuidanceCommand = {
                 }
                 const stats = fs.statSync(filePath);
                 const relativePath = filePath.replace(cwd, '').replace(/\\/g, '/');
-                // Delete old chunks
+                // Delete old chunks. Also delete any legacy doc-* row (#1053 S4).
                 db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${chunkPrefix}%`]);
-                // Store full document
-                const docMetadata = {
-                    type: 'document',
-                    filePath: relativePath,
-                    fileSize: stats.size,
-                    lastModified: stats.mtime.toISOString(),
-                    contentHash: contentHash_,
-                    indexedAt: new Date().toISOString(),
-                    ragVersion: '2.0',
-                };
-                batchStoreEntry(db, docKey, NAMESPACE, content, docMetadata, [keyPrefix, 'document']);
+                db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, docKey]);
+                // #1053 S4: doc-* entries no longer written. parentDoc on chunks
+                // remains as an identifier label; callers Read parentPath when
+                // they need the source file (see shipped/moflo-memory-protocol.md).
                 // Chunk content
                 const chunks = chunkMarkdown(content, fileName);
                 if (chunks.length === 0) {
@@ -1650,26 +1644,24 @@ const indexGuidanceCommand = {
                 }
                 const hierarchy = buildHierarchy(chunks, chunkPrefix);
                 const siblings = chunks.map((_, i) => `${chunkPrefix}-${i}`);
-                // Update doc with children refs
-                const docChildrenMeta = { ...docMetadata, children: siblings, chunkCount: chunks.length };
-                batchStoreEntry(db, docKey, NAMESPACE, content, docChildrenMeta, [keyPrefix, 'document']);
                 for (let i = 0; i < chunks.length; i++) {
                     const chunk = chunks[i];
                     const chunkKey = `${chunkPrefix}-${i}`;
                     const prevChunk = i > 0 ? `${chunkPrefix}-${i - 1}` : null;
                     const nextChunk = i < chunks.length - 1 ? `${chunkPrefix}-${i + 1}` : null;
-                    const contextBefore = i > 0
-                        ? extractOverlapContext(chunks[i - 1].content, overlapPercent, 'end')
-                        : null;
-                    const contextAfter = i < chunks.length - 1
-                        ? extractOverlapContext(chunks[i + 1].content, overlapPercent, 'start')
-                        : null;
+                    // #1053 S5: dropped prev/next preamble wrapping. Traversal happens
+                    // via memory_get_neighbors now (S2).
                     const hierInfo = hierarchy[chunkKey];
                     const chunkMetadata = {
                         type: 'chunk',
                         ragVersion: '2.0',
+                        // #1053 S4: parentDoc is an identifier label (target row no
+                        // longer exists); use parentPath for the actual source file.
+                        // docContentHash on every chunk lets the skip-if-unchanged
+                        // check read it off chunk-0.
                         parentDoc: docKey,
                         parentPath: relativePath,
+                        docContentHash: contentHash_,
                         chunkIndex: i,
                         totalChunks: chunks.length,
                         prevChunk,
@@ -1682,21 +1674,12 @@ const indexGuidanceCommand = {
                         headerLine: chunk.headerLine,
                         isPart: chunk.isPart || false,
                         partNum: chunk.partNum || null,
-                        contextOverlapPercent: overlapPercent,
-                        hasContextBefore: !!contextBefore,
-                        hasContextAfter: !!contextAfter,
                         contentLength: chunk.content.length,
                         contentHash: hashContent(chunk.content),
                         indexedAt: new Date().toISOString(),
                     };
-                    let searchableContent = `# ${chunk.title}\n\n`;
-                    if (contextBefore) {
-                        searchableContent += `[Context from previous section:]\n${contextBefore}\n\n---\n\n`;
-                    }
-                    searchableContent += chunk.content;
-                    if (contextAfter) {
-                        searchableContent += `\n\n---\n\n[Context from next section:]\n${contextAfter}`;
-                    }
+                    // #1053 S5: title heading + chunk body. No prev/next preamble.
+                    const searchableContent = `# ${chunk.title}\n\n${chunk.content}`;
                     batchStoreEntry(db, chunkKey, NAMESPACE, searchableContent, chunkMetadata, [keyPrefix, 'chunk', `level-${chunk.level}`, chunk.title.toLowerCase().replace(/[^a-z0-9]+/g, '-')]);
                 }
                 return { docKey, status: 'indexed', chunks: chunks.length };
@@ -1753,22 +1736,28 @@ const indexGuidanceCommand = {
                     errors++;
                 }
             }
-            // Clean stale entries for deleted files
-            const docsStmt = db.prepare(`SELECT DISTINCT key FROM memory_entries WHERE namespace = ? AND key LIKE 'doc-%'`);
-            docsStmt.bind([NAMESPACE]);
-            const docs = [];
-            while (docsStmt.step())
-                docs.push(docsStmt.getAsObject());
-            docsStmt.free();
-            for (const { key } of docs) {
-                if (!key.startsWith('doc-guidance-'))
-                    continue;
-                const checkPath = pathModule.resolve(cwd, '.claude/guidance', key.replace('doc-guidance-', '') + '.md');
+            // #1053 S4: Clean stale chunks for deleted files.
+            // doc-* markers are gone — derive prefixes from chunk keys directly.
+            // Chunk key shape: chunk-guidance-<filename>-<index>; group by stripping
+            // the trailing -<index>.
+            const chunksStmt = db.prepare(`SELECT DISTINCT key FROM memory_entries WHERE namespace = ? AND key LIKE 'chunk-guidance-%'`);
+            chunksStmt.bind([NAMESPACE]);
+            const seenPrefixes = new Set();
+            while (chunksStmt.step()) {
+                const { key } = chunksStmt.getAsObject();
+                const prefix = key.replace(/-\d+$/, '');
+                seenPrefixes.add(prefix);
+            }
+            chunksStmt.free();
+            for (const prefix of seenPrefixes) {
+                const filename = prefix.replace('chunk-guidance-', '') + '.md';
+                const checkPath = pathModule.resolve(cwd, '.claude/guidance', filename);
                 if (!fs.existsSync(checkPath)) {
-                    const cp = key.replace('doc-', 'chunk-');
-                    db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${cp}%`]);
-                    db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, key]);
-                    output.writeln(output.dim(`  Removed stale: ${key}`));
+                    db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${prefix}-%`]);
+                    // Also sweep any legacy doc-* row for this prefix (one-time tidy).
+                    const legacyDocKey = prefix.replace('chunk-', 'doc-');
+                    db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, legacyDocKey]);
+                    output.writeln(output.dim(`  Removed stale: ${prefix}-* (file ${filename} not found)`));
                 }
             }
         }

package/dist/src/cli/init/claudemd-generator.js

@@ -31,6 +31,10 @@ function mofloSection() {
 
 Your first tool call MUST be \`mcp__moflo__memory_search\` — before any Glob/Grep/Read. Search \`guidance\`, \`patterns\`, and \`learnings\` every prompt; add \`code-map\` when navigating code, \`tests\` when looking for test inventory or coverage. When the user says "remember this", call \`mcp__moflo__memory_store\` with namespace \`learnings\`.
 
+### Traverse chunks, don't bulk-retrieve
+
+Search results carry a compact \`navigation\` crumb (parentDoc, prev/next, chunkTitle). For adjacent/sibling/hierarchical context use \`mcp__moflo__memory_get_neighbors\`; for full chunk content use \`mcp__moflo__memory_retrieve\`; \`Read\` the source doc only via \`parentPath\` when truly needed. Full protocol: \`.claude/guidance/moflo-memory-protocol.md\`.
+
 ### Auto-enforced gates
 
 - **TaskCreate-first**: Call \`TaskCreate\` before spawning the Agent tool

package/dist/src/cli/init/moflo-init.js

@@ -435,7 +435,7 @@ function generateClaudeMd(root, _force) {
 // scriptFiles array in bin/session-start-launcher.mjs — first-init drops any
 // script missing here, and the launcher's manifest cleanup later treats it as
 // orphan residue and deletes it (#777, feedback_scriptfiles_sync.md).
-const SCRIPT_MAP = [
+export const SCRIPT_MAP = [
     'hooks.mjs',
     'session-start-launcher.mjs',
     'index-guidance.mjs',
@@ -496,17 +496,25 @@ function isStale(srcPath, destPath) {
 // ============================================================================
 // Step 6: .gitignore
 // ============================================================================
-function updateGitignore(root) {
+export function updateGitignore(root) {
     const gitignorePath = path.join(root, '.gitignore');
     const entries = [
         '.claude-epic/',
         '.moflo/',
         '.swarm/',
-        '.moflo/',
         '.claude/settings.local.json',
         '.claude/scheduled_tasks.lock',
         '**/workflow-state.json',
+        // Leading `/` anchors to gitignore root — bare `.claude/guidance/` once
+        // swallowed shipped/internal subdirs and broke `npm pack`
+        // (guidance-gitignore-shipped-trap).
+        '/.claude/guidance/moflo-*.md',
+        ...SCRIPT_MAP.map(name => `/.claude/scripts/${name}`),
     ];
+    // Treat `/.foo` and `.foo` as the same rule when checking for prior presence
+    // — both forms anchor at gitignore root, so a consumer who wrote either
+    // shouldn't get a duplicate appended.
+    const normalize = (s) => s.replace(/^\//, '');
     if (!fs.existsSync(gitignorePath)) {
         const defaultEntries = ['node_modules/', 'dist/', '.env', '.env.*', ''];
         const content = '# Dependencies\n' + defaultEntries.join('\n') + '\n# MoFlo state\n' + entries.join('\n') + '\n';
@@ -514,8 +522,8 @@ function updateGitignore(root) {
         return { name: '.gitignore', status: 'created', detail: 'Created with node_modules, .env, and MoFlo entries' };
     }
     const existing = fs.readFileSync(gitignorePath, 'utf-8');
-    const existingLines = new Set(existing.split(/\r?\n/).map(l => l.trim()).filter(l => l && !l.startsWith('#')));
-    const toAdd = entries.filter(e => !existingLines.has(e));
+    const existingLines = new Set(existing.split(/\r?\n/).map(l => normalize(l.trim())).filter(l => l && !l.startsWith('#')));
+    const toAdd = entries.filter(e => !existingLines.has(normalize(e)));
     if (toAdd.length === 0) {
         return { name: '.gitignore', status: 'skipped', detail: 'Entries already present' };
     }