moflo 4.8.41 → 4.8.43

package/.claude/scripts/index-all.mjs

@@ -1,127 +1,183 @@
-#!/usr/bin/env node
-/**
- * Sequential indexer chain for session-start.
- *
- * Runs all DB-writing indexers one at a time to avoid sql.js last-write-wins
- * concurrency issues, then triggers HNSW rebuild once everything is committed.
- *
- * Spawned as a single detached background process by hooks.mjs session-start.
- */
-
-import { existsSync, appendFileSync } from 'fs';
-import { resolve, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { execFileSync } from 'child_process';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const projectRoot = resolve(__dirname, '../..');
-const LOG_PATH = resolve(projectRoot, '.claude/hooks.log');
-
-function log(msg) {
-  const ts = new Date().toISOString().replace('T', ' ').slice(0, 19);
-  const line = `[${ts}] [index-all] ${msg}\n`;
-  try { appendFileSync(LOG_PATH, line); } catch { /* ignore */ }
-}
-
-function resolveBin(binName, localScript) {
-  const mofloScript = resolve(projectRoot, 'node_modules/moflo/bin', localScript);
-  if (existsSync(mofloScript)) return mofloScript;
-  const npmBin = resolve(projectRoot, 'node_modules/.bin', binName);
-  if (existsSync(npmBin)) return npmBin;
-  const localPath = resolve(projectRoot, '.claude/scripts', localScript);
-  if (existsSync(localPath)) return localPath;
-  return null;
-}
-
-function getLocalCliPath() {
-  const paths = [
-    resolve(projectRoot, 'node_modules/moflo/src/@claude-flow/cli/bin/cli.js'),
-    resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
-    resolve(projectRoot, 'node_modules/.bin/flo'),
-    resolve(projectRoot, 'src/@claude-flow/cli/bin/cli.js'),
-  ];
-  for (const p of paths) {
-    if (existsSync(p)) return p;
-  }
-  return null;
-}
-
-function runStep(label, cmd, args, timeoutMs = 120_000) {
-  const start = Date.now();
-  log(`START ${label}`);
-  try {
-    execFileSync(cmd, args, {
-      cwd: projectRoot,
-      timeout: timeoutMs,
-      stdio: 'ignore',
-      windowsHide: true,
-    });
-    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-    log(`DONE  ${label} (${elapsed}s)`);
-    return true;
-  } catch (err) {
-    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-    log(`FAIL  ${label} (${elapsed}s): ${err.message?.split('\n')[0] || 'unknown'}`);
-    return false;
-  }
-}
-
-async function main() {
-  const startTime = Date.now();
-  log('Sequential indexing chain started');
-
-  // 1. Guidance indexer
-  const guidanceScript = resolveBin('flo-index', 'index-guidance.mjs');
-  if (guidanceScript) {
-    runStep('guidance-index', 'node', [guidanceScript]);
-  } else {
-    log('SKIP  guidance-index (script not found)');
-  }
-
-  // 2. Code map generator (the big one — ~22s)
-  const codeMapScript = resolveBin('flo-codemap', 'generate-code-map.mjs');
-  if (codeMapScript) {
-    runStep('code-map', 'node', [codeMapScript], 180_000);
-  } else {
-    log('SKIP  code-map (script not found)');
-  }
-
-  // 3. Test indexer
-  const testScript = resolveBin('flo-testmap', 'index-tests.mjs');
-  if (testScript) {
-    runStep('test-index', 'node', [testScript]);
-  } else {
-    log('SKIP  test-index (script not found)');
-  }
-
-  // 4. Patterns indexer
-  const patternsScript = resolveBin('flo-patterns', 'index-patterns.mjs');
-  if (patternsScript) {
-    runStep('patterns-index', 'node', [patternsScript]);
-  } else {
-    log('SKIP  patterns-index (script not found)');
-  }
-
-  // 5. Pretrain (extracts patterns from repository)
-  const localCli = getLocalCliPath();
-  if (localCli) {
-    runStep('pretrain', 'node', [localCli, 'hooks', 'pretrain']);
-  } else {
-    log('SKIP  pretrain (CLI not found)');
-  }
-
-  // 6. HNSW rebuild — MUST run last, after all writes are committed (#81)
-  if (localCli) {
-    runStep('hnsw-rebuild', 'node', [localCli, 'memory', 'rebuild', '--force']);
-  } else {
-    log('SKIP  hnsw-rebuild (CLI not found)');
-  }
-
-  const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-  log(`Sequential indexing chain complete (${totalElapsed}s)`);
-}
-
-main().catch(err => {
-  log(`FATAL: ${err.message}`);
-  process.exit(1);
-});
+#!/usr/bin/env node
+/**
+ * Sequential indexer chain for session-start.
+ *
+ * Runs all DB-writing indexers one at a time to avoid sql.js last-write-wins
+ * concurrency issues (#78), then triggers HNSW rebuild once everything is
+ * committed (#81).
+ *
+ * Spawned as a single detached background process by hooks.mjs session-start.
+ */
+
+import { existsSync, appendFileSync, readFileSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { execFileSync } from 'child_process';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Detect project root by walking up from cwd to find package.json.
+// IMPORTANT: Do NOT use resolve(__dirname, '..') — this script lives in bin/
+// during development but gets synced to .claude/scripts/ in consumer projects,
+// so __dirname-relative paths break. findProjectRoot() works in both locations.
+function findProjectRoot() {
+  let dir = process.cwd();
+  const root = resolve(dir, '/');
+  while (dir !== root) {
+    if (existsSync(resolve(dir, 'package.json'))) return dir;
+    dir = dirname(dir);
+  }
+  return process.cwd();
+}
+
+const projectRoot = findProjectRoot();
+const LOG_PATH = resolve(projectRoot, '.swarm/hooks.log');
+
+function log(msg) {
+  const ts = new Date().toISOString().replace('T', ' ').slice(0, 19);
+  const line = `[${ts}] [index-all] ${msg}\n`;
+  try { appendFileSync(LOG_PATH, line); } catch { /* ignore */ }
+}
+
+function resolveBin(binName, localScript) {
+  const mofloScript = resolve(projectRoot, 'node_modules/moflo/bin', localScript);
+  if (existsSync(mofloScript)) return mofloScript;
+  const npmBin = resolve(projectRoot, 'node_modules/.bin', binName);
+  if (existsSync(npmBin)) return npmBin;
+  const localPath = resolve(projectRoot, '.claude/scripts', localScript);
+  if (existsSync(localPath)) return localPath;
+  // Also check bin/ directory (for development use)
+  const binPath = resolve(projectRoot, 'bin', localScript);
+  if (existsSync(binPath)) return binPath;
+  return null;
+}
+
+function getLocalCliPath() {
+  const paths = [
+    resolve(projectRoot, 'node_modules/moflo/src/@claude-flow/cli/bin/cli.js'),
+    resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
+    resolve(projectRoot, 'node_modules/.bin/flo'),
+    // Development: local CLI
+    resolve(projectRoot, 'src/@claude-flow/cli/bin/cli.js'),
+  ];
+  for (const p of paths) {
+    if (existsSync(p)) return p;
+  }
+  return null;
+}
+
+/** Read moflo.yaml once and cache auto_index flags. */
+let _autoIndexFlags = null;
+function isIndexEnabled(key) {
+  if (_autoIndexFlags === null) {
+    _autoIndexFlags = {};
+    const yamlPath = resolve(projectRoot, 'moflo.yaml');
+    if (existsSync(yamlPath)) {
+      try {
+        const content = readFileSync(yamlPath, 'utf-8');
+        for (const k of ['guidance', 'code_map', 'tests', 'patterns']) {
+          const re = new RegExp(`auto_index:\\s*\\n(?:.*\\n)*?\\s+${k}:\\s*(true|false)`);
+          const match = content.match(re);
+          _autoIndexFlags[k] = match ? match[1] !== 'false' : true;
+        }
+      } catch { /* ignore, all default to true */ }
+    }
+  }
+  return _autoIndexFlags[key] !== false;
+}
+
+function runStep(label, cmd, args, timeoutMs = 120_000) {
+  const start = Date.now();
+  log(`START ${label}`);
+  try {
+    execFileSync(cmd, args, {
+      cwd: projectRoot,
+      timeout: timeoutMs,
+      stdio: 'ignore',
+      windowsHide: true,
+    });
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+    log(`DONE  ${label} (${elapsed}s)`);
+    return true;
+  } catch (err) {
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+    log(`FAIL  ${label} (${elapsed}s): ${err.message?.split('\n')[0] || 'unknown'}`);
+    return false;
+  }
+}
+
+async function main() {
+  const startTime = Date.now();
+  log('Sequential indexing chain started');
+
+  // 1. Guidance indexer
+  if (isIndexEnabled('guidance')) {
+    const guidanceScript = resolveBin('flo-index', 'index-guidance.mjs');
+    if (guidanceScript) {
+      runStep('guidance-index', 'node', [guidanceScript]);
+    } else {
+      log('SKIP  guidance-index (script not found)');
+    }
+  } else {
+    log('SKIP  guidance-index (disabled in moflo.yaml)');
+  }
+
+  // 2. Code map generator (the big one — ~22s)
+  if (isIndexEnabled('code_map')) {
+    const codeMapScript = resolveBin('flo-codemap', 'generate-code-map.mjs');
+    if (codeMapScript) {
+      runStep('code-map', 'node', [codeMapScript], 180_000);
+    } else {
+      log('SKIP  code-map (script not found)');
+    }
+  } else {
+    log('SKIP  code-map (disabled in moflo.yaml)');
+  }
+
+  // 3. Test indexer
+  if (isIndexEnabled('tests')) {
+    const testScript = resolveBin('flo-testmap', 'index-tests.mjs');
+    if (testScript) {
+      runStep('test-index', 'node', [testScript]);
+    } else {
+      log('SKIP  test-index (script not found)');
+    }
+  } else {
+    log('SKIP  test-index (disabled in moflo.yaml)');
+  }
+
+  // 4. Patterns indexer
+  if (isIndexEnabled('patterns')) {
+    const patternsScript = resolveBin('flo-patterns', 'index-patterns.mjs');
+    if (patternsScript) {
+      runStep('patterns-index', 'node', [patternsScript]);
+    } else {
+      log('SKIP  patterns-index (script not found)');
+    }
+  } else {
+    log('SKIP  patterns-index (disabled in moflo.yaml)');
+  }
+
+  // 5. Pretrain (extracts patterns from repository)
+  const localCli = getLocalCliPath();
+  if (localCli) {
+    runStep('pretrain', 'node', [localCli, 'hooks', 'pretrain']);
+  } else {
+    log('SKIP  pretrain (CLI not found)');
+  }
+
+  // 6. HNSW rebuild — MUST run last, after all writes are committed (#81)
+  if (localCli) {
+    runStep('hnsw-rebuild', 'node', [localCli, 'memory', 'rebuild', '--force']);
+  } else {
+    log('SKIP  hnsw-rebuild (CLI not found)');
+  }
+
+  const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  log(`Sequential indexing chain complete (${totalElapsed}s)`);
+}
+
+main().catch(err => {
+  log(`FATAL: ${err.message}`);
+  process.exit(1);
+});

package/.claude/scripts/index-guidance.mjs

@@ -819,8 +819,17 @@ if (!specificFile) {
   }
 }
 
-// Write changes back to disk and close
+// Write changes back to disk
 if (docsIndexed > 0 || chunksIndexed > 0 || staleRemoved > 0) saveDb(db);
+
+// Check for entries missing embeddings (e.g. prior background run failed)
+let missingEmbeddings = 0;
+{
+  const stmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE namespace = ? AND (embedding IS NULL OR embedding = '')`);
+  stmt.bind([NAMESPACE]);
+  if (stmt.step()) missingEmbeddings = stmt.getAsObject().cnt;
+  stmt.free();
+}
 db.close();
 
 console.log('');
@@ -840,9 +849,13 @@ log(`    • Hierarchical links (h2 -> h3 parent/children)`);
 log(`    • Context overlap: ${overlapPercent}% (contextBefore/contextAfter)`);
 log('═══════════════════════════════════════════════════════════');
 
-// Generate embeddings for new entries (unless skipped or nothing changed)
+// Generate embeddings for new entries or backfill missing ones
 // Runs in BACKGROUND to avoid blocking startup
-if (!skipEmbeddings && (docsIndexed > 0 || chunksIndexed > 0)) {
+const needsEmbeddings = (docsIndexed > 0 || chunksIndexed > 0 || missingEmbeddings > 0);
+if (!skipEmbeddings && needsEmbeddings) {
+  if (missingEmbeddings > 0 && docsIndexed === 0 && chunksIndexed === 0) {
+    log(`${missingEmbeddings} entries missing embeddings — backfilling...`);
+  }
   console.log('');
   log('Spawning embedding generation in background...');
 

package/.claude/scripts/index-tests.mjs

@@ -153,6 +153,15 @@ function countNamespace(db) {
   return count;
 }
 
+function countMissingEmbeddings(db) {
+  const stmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE namespace = ? AND (embedding IS NULL OR embedding = '')`);
+  stmt.bind([NAMESPACE]);
+  let count = 0;
+  if (stmt.step()) count = stmt.getAsObject().cnt;
+  stmt.free();
+  return count;
+}
+
 // ---------------------------------------------------------------------------
 // Test directory discovery
 // ---------------------------------------------------------------------------
@@ -618,9 +627,15 @@ async function main() {
   if (isUnchanged(currentHash)) {
     const db = await getDb();
     const count = countNamespace(db);
+    const missing = countMissingEmbeddings(db);
     db.close();
     if (count > 0) {
-      log(`Skipping — file list unchanged (${count} chunks in DB, hash ${currentHash.slice(0, 12)}...)`);
+      if (missing > 0 && !skipEmbeddings) {
+        log(`File list unchanged but ${missing}/${count} entries missing embeddings — generating...`);
+        await runEmbeddings();
+      } else {
+        log(`Skipping — file list unchanged (${count} chunks in DB, hash ${currentHash.slice(0, 12)}...)`);
+      }
       return;
     }
     log('File list unchanged but no chunks in DB — forcing regeneration');
@@ -683,24 +698,28 @@ async function main() {
 
   // 7. Generate embeddings (inline, like code-map)
   if (!skipEmbeddings && allChunks.length > 0) {
-    const embedCandidates = [
-      resolve(dirname(fileURLToPath(import.meta.url)), 'build-embeddings.mjs'),
-      resolve(projectRoot, '.claude/scripts/build-embeddings.mjs'),
-    ];
-    const embedScript = embedCandidates.find(p => existsSync(p));
-    if (embedScript) {
-      log('Generating embeddings for tests...');
-      try {
-        execSync(`node "${embedScript}" --namespace tests`, {
-          cwd: projectRoot,
-          stdio: 'inherit',
-          timeout: 120000,
-          windowsHide: true,
-        });
-      } catch (err) {
-        log(`Warning: embedding generation failed: ${err.message?.split('\n')[0]}`);
-      }
-    }
+    await runEmbeddings();
+  }
+}
+
+async function runEmbeddings() {
+  const embedCandidates = [
+    resolve(dirname(fileURLToPath(import.meta.url)), 'build-embeddings.mjs'),
+    resolve(projectRoot, '.claude/scripts/build-embeddings.mjs'),
+  ];
+  const embedScript = embedCandidates.find(p => existsSync(p));
+  if (!embedScript) return;
+
+  log('Generating embeddings for tests...');
+  try {
+    execSync(`node "${embedScript}" --namespace tests`, {
+      cwd: projectRoot,
+      stdio: 'inherit',
+      timeout: 120000,
+      windowsHide: true,
+    });
+  } catch (err) {
+    log(`Warning: embedding generation failed: ${err.message?.split('\n')[0]}`);
   }
 }
 

package/.claude/scripts/lib/moflo-resolve.mjs

@@ -0,0 +1,14 @@
+/**
+ * Shared dependency resolver for moflo bin scripts.
+ * Resolves packages from moflo's own node_modules (not the consuming project's).
+ * On Windows, converts native paths to file:// URLs required by ESM import().
+ */
+
+import { createRequire } from 'module';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+const __require = createRequire(fileURLToPath(import.meta.url));
+
+export function mofloResolveURL(specifier) {
+  return pathToFileURL(__require.resolve(specifier)).href;
+}