moflo 4.9.36 → 4.10.0

package/bin/index-guidance.mjs

@@ -25,25 +25,15 @@
 import { existsSync, readdirSync, readFileSync, statSync, mkdirSync, writeFileSync } from 'fs';
 import { resolve, relative, dirname, basename, extname } from 'path';
 import { fileURLToPath } from 'url';
-import { mofloResolveURL } from './lib/moflo-resolve.mjs';
-import { memoryDbPath } from './lib/moflo-paths.mjs';
+import { memoryDbPath, findProjectRoot } from './lib/moflo-paths.mjs';
+import { openBackend } from './lib/get-backend.mjs';
+import { applyIncrementalChunks } from './lib/incremental-write.mjs';
 import { resolveMofloBin } from './lib/resolve-bin.mjs';
 import { createProcessManager } from './lib/process-manager.mjs';
-const initSqlJs = (await import(mofloResolveURL('sql.js'))).default;
 
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-function findProjectRoot() {
-  let dir = process.cwd();
-  const root = resolve(dir, '/');
-  while (dir !== root) {
-    if (existsSync(resolve(dir, 'package.json'))) return dir;
-    dir = dirname(dir);
-  }
-  return process.cwd();
-}
-
 const projectRoot = findProjectRoot();
 
 // Locate the moflo package root (for bundled guidance that ships with moflo)
@@ -181,14 +171,7 @@ function ensureDbDir() {
 
 async function getDb() {
   ensureDbDir();
-  const SQL = await initSqlJs();
-  let db;
-  if (existsSync(DB_PATH)) {
-    const buffer = readFileSync(DB_PATH);
-    db = new SQL.Database(buffer);
-  } else {
-    db = new SQL.Database();
-  }
+  const db = await openBackend(projectRoot, { create: true });
 
   // Ensure table exists with unique constraint
   db.run(`
@@ -221,12 +204,7 @@ async function getDb() {
 }
 
 function saveDb(db) {
-  const data = db.export();
-  writeFileSync(DB_PATH, Buffer.from(data));
-}
-
-function generateId() {
-  return `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  db.save();
 }
 
 function hashContent(content) {
@@ -239,34 +217,17 @@ function hashContent(content) {
   return hash.toString(16);
 }
 
-function storeEntry(db, key, content, metadata = {}, tags = []) {
-  const now = Date.now();
-  const id = generateId();
-  const metaJson = JSON.stringify(metadata);
-  const tagsJson = JSON.stringify(tags);
-
-  db.run(`
-    INSERT OR REPLACE INTO memory_entries
-    (id, key, namespace, content, metadata, tags, created_at, updated_at, status)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active')
-  `, [id, key, NAMESPACE, content, metaJson, tagsJson, now, now]);
-
-  return true;
-}
-
-function deleteByPrefix(db, prefix) {
-  db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${prefix}%`]);
-}
-
-function getEntryHash(db, key) {
+// #1053 S4: doc-* entries retired. Doc-level skip check now reads
+// docContentHash off chunk-0 (every chunk carries it).
+function getDocHashFromChunkZero(db, chunkPrefix) {
   const stmt = db.prepare('SELECT metadata FROM memory_entries WHERE key = ? AND namespace = ?');
-  stmt.bind([key, NAMESPACE]);
+  stmt.bind([`${chunkPrefix}-0`, NAMESPACE]);
   const entry = stmt.step() ? stmt.getAsObject() : null;
   stmt.free();
   if (entry?.metadata) {
     try {
       const meta = JSON.parse(entry.metadata);
-      return meta.contentHash;
+      return meta.docContentHash;
     } catch { /* ignore */ }
   }
   return null;
@@ -536,9 +497,10 @@ function indexFile(db, filePath, keyPrefix, options = {}) {
     const content = readFileSync(filePath, 'utf-8');
     const contentHash = hashContent(content);
 
-    // Check if content changed (skip if same hash unless --force)
+    // Check if content changed (skip if same hash unless --force).
+    // #1053 S4: doc-* retired — read docContentHash off chunk-0 instead.
     if (!force) {
-      const existingHash = getEntryHash(db, docKey);
+      const existingHash = getDocHashFromChunkZero(db, chunkPrefix);
       if (existingHash === contentHash) {
         return { docKey, status: 'unchanged', chunks: 0 };
       }
@@ -547,28 +509,23 @@ function indexFile(db, filePath, keyPrefix, options = {}) {
     const stats = statSync(filePath);
     const relativePath = '/' + relative(projectRoot, filePath).replace(/\\/g, '/');
 
-    // Delete old chunks for this file before re-indexing
-    deleteByPrefix(db, chunkPrefix);
-
-    // 1. Store full document
-    const docMetadata = {
-      ...extraMetadata,
-      type: 'document',
-      filePath: relativePath,
-      fileSize: stats.size,
-      lastModified: stats.mtime.toISOString(),
-      contentHash,
-      indexedAt: new Date().toISOString(),
-      ragVersion: '2.0',  // Mark as full RAG indexed
-    };
+    // #1053 S4: drop any legacy doc-* row left over from a pre-S4 install.
+    // The chunker stopped emitting these and the audit found zero production
+    // readers; safe one-time cleanup as part of the per-doc re-index.
+    db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key = ?`, [NAMESPACE, docKey]);
 
-    storeEntry(db, docKey, content, docMetadata, [keyPrefix, 'document', ...extraTags]);
-    debug(`Stored document: ${docKey}`);
+    // #1053 S4: Chunker no longer writes doc-* entries. Audit found zero
+    // production readers — they duplicated chunk semantic territory and
+    // ate ~13% of search slots without unique signal. parentDoc on chunks
+    // remains as an identifier/grouping label; callers needing the source
+    // file Read parentPath, per shipped/moflo-memory-protocol.md.
 
-    // 2. Chunk and store semantic pieces with full RAG linking
+    // Chunk and store semantic pieces with full RAG linking
     const chunks = chunkMarkdown(content, fileName);
 
     if (chunks.length === 0) {
+      // No chunks generated → sweep any stragglers from a prior run and exit.
+      db.run(`DELETE FROM memory_entries WHERE namespace = ? AND key LIKE ?`, [NAMESPACE, `${chunkPrefix}-%`]);
       return { docKey, status: 'indexed', chunks: 0 };
     }
 
@@ -576,31 +533,17 @@ function indexFile(db, filePath, keyPrefix, options = {}) {
     const hierarchy = buildHierarchy(chunks, chunkPrefix);
     const siblings = chunks.map((_, i) => `${chunkPrefix}-${i}`);
 
-    // Update document with children references
-    const docChildrenMeta = {
-      ...docMetadata,
-      children: siblings,
-      chunkCount: chunks.length,
-    };
-    storeEntry(db, docKey, content, docChildrenMeta, [keyPrefix, 'document', ...extraTags]);
-
-    for (let i = 0; i < chunks.length; i++) {
-      const chunk = chunks[i];
+    // #1057: route chunk writes through applyIncrementalChunks so an
+    // unchanged chunk (~95% of the time when only one section of a doc
+    // edited) keeps its embedding column. Pre-#1057 this loop used raw
+    // INSERT OR REPLACE after a blanket deleteByPrefix — every re-index
+    // wiped every chunk's embedding and forced build-embeddings to
+    // re-vectorise the whole doc on every save. See
+    // feedback_indexer_preserve_embeddings.md.
+    const chunkRows = chunks.map((chunk, i) => {
       const chunkKey = `${chunkPrefix}-${i}`;
-
-      // Build prev/next links
       const prevChunk = i > 0 ? `${chunkPrefix}-${i - 1}` : null;
       const nextChunk = i < chunks.length - 1 ? `${chunkPrefix}-${i + 1}` : null;
-
-      // Extract overlapping context from adjacent chunks
-      const contextBefore = i > 0
-        ? extractOverlapContext(chunks[i - 1].content, overlapPercent, 'end')
-        : null;
-      const contextAfter = i < chunks.length - 1
-        ? extractOverlapContext(chunks[i + 1].content, overlapPercent, 'start')
-        : null;
-
-      // Get hierarchical relationships
       const hierInfo = hierarchy[chunkKey];
 
       const chunkMetadata = {
@@ -608,9 +551,13 @@ function indexFile(db, filePath, keyPrefix, options = {}) {
         type: 'chunk',
         ragVersion: '2.0',
 
-        // Document relationship
+        // Document relationship — parentDoc is an identifier/grouping label
+        // only after #1053 S4; the actual source doc is at parentPath.
+        // docContentHash is the file-level hash, used by the skip-if-unchanged
+        // check (the chunker reads it off chunk-0 to decide whether to re-index).
         parentDoc: docKey,
         parentPath: relativePath,
+        docContentHash: contentHash,
 
         // Sequential navigation (forward/backward links)
         chunkIndex: i,
@@ -632,41 +579,34 @@ function indexFile(db, filePath, keyPrefix, options = {}) {
         isPart: chunk.isPart || false,
         partNum: chunk.partNum || null,
 
-        // Overlapping context for continuity
-        contextOverlapPercent: overlapPercent,
-        hasContextBefore: !!contextBefore,
-        hasContextAfter: !!contextAfter,
-
         // Content metadata
         contentLength: chunk.content.length,
         contentHash: hashContent(chunk.content),
         indexedAt: new Date().toISOString(),
       };
 
-      // Build searchable content with title context
-      // Include overlap context for better retrieval
-      let searchableContent = `# ${chunk.title}\n\n`;
-
-      if (contextBefore) {
-        searchableContent += `[Context from previous section:]\n${contextBefore}\n\n---\n\n`;
-      }
-
-      searchableContent += chunk.content;
-
-      if (contextAfter) {
-        searchableContent += `\n\n---\n\n[Context from next section:]\n${contextAfter}`;
-      }
-
-      // Store chunk with full metadata
-      storeEntry(
-        db,
-        chunkKey,
-        searchableContent,
-        chunkMetadata,
-        [keyPrefix, 'chunk', `level-${chunk.level}`, chunk.title.toLowerCase().replace(/[^a-z0-9]+/g, '-'), ...extraTags]
-      );
+      // #1053 S5: title heading + chunk body. No prev/next preamble.
+      const searchableContent = `# ${chunk.title}\n\n${chunk.content}`;
+
+      return {
+        key: chunkKey,
+        content: searchableContent,
+        metadata: chunkMetadata,
+        tags: [
+          keyPrefix,
+          'chunk',
+          `level-${chunk.level}`,
+          chunk.title.toLowerCase().replace(/[^a-z0-9]+/g, '-'),
+          ...extraTags,
+        ],
+      };
+    });
 
-      debug(`  Stored chunk ${i}: ${chunk.title} (${chunk.content.length} chars, prev=${!!prevChunk}, next=${!!nextChunk})`);
+    const counts = applyIncrementalChunks(db, NAMESPACE, chunkRows, {
+      keyPrefix: `${chunkPrefix}-`,
+    });
+    if (verbose) {
+      debug(`  Doc ${docKey}: inserted=${counts.inserted} updated=${counts.updated} unchanged=${counts.unchanged} removed=${counts.removed}`);
     }
 
     return { docKey, status: 'indexed', chunks: chunks.length };

package/bin/index-patterns.mjs

@@ -28,23 +28,13 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from
 import { resolve, dirname, relative, basename, extname } from 'path';
 import { fileURLToPath } from 'url';
 import { resolveMofloBin } from './lib/resolve-bin.mjs';
-import { mofloResolveURL } from './lib/moflo-resolve.mjs';
-import { memoryDbPath, MOFLO_DIR } from './lib/moflo-paths.mjs';
+import { memoryDbPath, MOFLO_DIR, findProjectRoot } from './lib/moflo-paths.mjs';
+import { openBackend } from './lib/get-backend.mjs';
 import { applyIncrementalChunks, computeContentListHash } from './lib/incremental-write.mjs';
 import { createProcessManager } from './lib/process-manager.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-function findProjectRoot() {
-  let dir = process.cwd();
-  const root = resolve(dir, '/');
-  while (dir !== root) {
-    if (existsSync(resolve(dir, 'package.json'))) return dir;
-    dir = dirname(dir);
-  }
-  return process.cwd();
-}
-
 const projectRoot = findProjectRoot();
 const NAMESPACE = 'patterns';
 const DB_PATH = memoryDbPath(projectRoot);
@@ -76,16 +66,9 @@ function ensureDbDir() {
 async function getDb() {
   ensureDbDir();
   // Lazy: hash-cache-match and no-source-files early-exits in main() never
-  // reach this, and the sql.js wasm cold-load is ~400ms otherwise wasted.
-  const initSqlJs = (await import(mofloResolveURL('sql.js'))).default;
-  const SQL = await initSqlJs();
-  let db;
-  if (existsSync(DB_PATH)) {
-    const buffer = readFileSync(DB_PATH);
-    db = new SQL.Database(buffer);
-  } else {
-    db = new SQL.Database();
-  }
+  // reach this, and the backend cold-load (sql.js wasm ~400ms, node:sqlite
+  // WAL init ~20ms) is otherwise wasted on the no-op path.
+  const db = await openBackend(projectRoot, { create: true });
   db.run(`
     CREATE TABLE IF NOT EXISTS memory_entries (
       id TEXT PRIMARY KEY,
@@ -114,8 +97,7 @@ async function getDb() {
 }
 
 function saveDb(db) {
-  const data = db.export();
-  writeFileSync(DB_PATH, Buffer.from(data));
+  db.save();
 }
 
 function countNamespace(db) {

package/bin/index-tests.mjs

@@ -26,24 +26,13 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from
 import { resolve, dirname, relative, basename, extname, join } from 'path';
 import { fileURLToPath } from 'url';
 import { execSync, execFileSync, spawn } from 'child_process';
-import { mofloResolveURL } from './lib/moflo-resolve.mjs';
-import { memoryDbPath, MOFLO_DIR } from './lib/moflo-paths.mjs';
+import { memoryDbPath, MOFLO_DIR, findProjectRoot } from './lib/moflo-paths.mjs';
+import { openBackend } from './lib/get-backend.mjs';
 import { resolveMofloBin } from './lib/resolve-bin.mjs';
 import { applyIncrementalChunks, computeContentListHash } from './lib/incremental-write.mjs';
-const initSqlJs = (await import(mofloResolveURL('sql.js'))).default;
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-function findProjectRoot() {
-  let dir = process.cwd();
-  const root = resolve(dir, '/');
-  while (dir !== root) {
-    if (existsSync(resolve(dir, 'package.json'))) return dir;
-    dir = dirname(dir);
-  }
-  return process.cwd();
-}
-
 const projectRoot = findProjectRoot();
 const NAMESPACE = 'tests';
 const DB_PATH = memoryDbPath(projectRoot);
@@ -87,14 +76,7 @@ function ensureDbDir() {
 
 async function getDb() {
   ensureDbDir();
-  const SQL = await initSqlJs();
-  let db;
-  if (existsSync(DB_PATH)) {
-    const buffer = readFileSync(DB_PATH);
-    db = new SQL.Database(buffer);
-  } else {
-    db = new SQL.Database();
-  }
+  const db = await openBackend(projectRoot, { create: true });
 
   db.run(`
     CREATE TABLE IF NOT EXISTS memory_entries (
@@ -124,8 +106,7 @@ async function getDb() {
 }
 
 function saveDb(db) {
-  const data = db.export();
-  writeFileSync(DB_PATH, Buffer.from(data));
+  db.save();
 }
 
 function countNamespace(db) {

package/bin/lib/db-repair.mjs

@@ -24,20 +24,9 @@
  * the DB and BEFORE the embeddings migration / soft-delete purge / ephemeral
  * purge — those all swallow corruption errors and silently no-op.
  */
-import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { existsSync } from 'node:fs';
 import { memoryDbPath } from './moflo-paths.mjs';
-
-let _initSqlJs = null;
-
-async function loadSqlJs() {
-  if (_initSqlJs) return _initSqlJs;
-  // sql.js is a hard dependency of moflo (see top-level package.json);
-  // resolving it from the consumer's node_modules works because the launcher
-  // runs from the consumer cwd.
-  const mod = await import('sql.js');
-  _initSqlJs = mod.default || mod;
-  return _initSqlJs;
-}
+import { openBackend } from './get-backend.mjs';
 
 function isOk(execResult) {
   const rows = execResult?.[0]?.values ?? [];
@@ -63,18 +52,9 @@ export async function repairMemoryDbIfCorrupt(projectRoot) {
   const dbPath = memoryDbPath(projectRoot);
   if (!existsSync(dbPath)) return { repaired: false, errors: 0 };
 
-  let initSql;
-  try {
-    initSql = await loadSqlJs();
-  } catch {
-    return { repaired: false, errors: 0 };
-  }
-
   let db = null;
   try {
-    const SQL = await initSql();
-    const data = readFileSync(dbPath);
-    db = new SQL.Database(data);
+    db = await openBackend(projectRoot, { create: false });
 
     const before = db.exec('PRAGMA integrity_check');
     if (isOk(before)) {
@@ -89,8 +69,7 @@ export async function repairMemoryDbIfCorrupt(projectRoot) {
       return { repaired: false, errors, persistent: true };
     }
 
-    const out = Buffer.from(db.export());
-    writeFileSync(dbPath, out);
+    db.save();
     return { repaired: true, errors };
   } catch {
     return { repaired: false, errors: 0 };