moflo 4.8.42 → 4.8.44

package/.claude/scripts/hooks.mjs

@@ -27,7 +27,22 @@ import { createProcessManager } from './lib/process-manager.mjs';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const projectRoot = resolve(__dirname, '../..');
+
+// Detect project root by walking up from cwd to find package.json.
+// IMPORTANT: Do NOT use resolve(__dirname, '..') or '../..' — this script lives
+// in bin/ during development but gets synced to .claude/scripts/ in consumer
+// projects, so __dirname-relative paths break. findProjectRoot() works everywhere.
+function findProjectRoot() {
+  let dir = process.cwd();
+  const root = resolve(dir, '/');
+  while (dir !== root) {
+    if (existsSync(resolve(dir, 'package.json'))) return dir;
+    dir = dirname(dir);
+  }
+  return process.cwd();
+}
+
+const projectRoot = findProjectRoot();
 const logFile = resolve(projectRoot, '.swarm/hooks.log');
 const pm = createProcessManager(projectRoot);
 
@@ -260,21 +275,14 @@ async function main() {
       case 'session-start': {
         // Start daemon quietly in background (no DB writes)
         runDaemonStartBackground();
-        // Run pretrain in background (writes to patterns namespace, fast)
-        runBackgroundPretrain();
+        // Initialize embeddings engine (must run before indexers that generate embeddings)
+        runEmbeddingsInitBackground();
         // Run all DB-writing indexers SEQUENTIALLY in a single background process.
-        // This avoids sql.js last-write-wins concurrency (issue #78).
-        // Chain: guidance → code-map → tests → HNSW rebuild (must be last).
-        const indexAllScript = resolve(__dirname, 'index-all.mjs');
-        if (existsSync(indexAllScript)) {
-          spawnWindowless('node', [indexAllScript], 'sequential indexing chain');
-        } else {
-          // Fallback to parallel if index-all.mjs not available
-          runIndexGuidanceBackground();
-          runCodeMapBackground();
-          runTestIndexBackground();
-          runHNSWRebuildBackground();
-        }
+        // This avoids sql.js last-write-wins concurrency (#78) and ensures
+        // HNSW rebuild runs after all indexers finish (#81).
+        // Chain: guidance → code-map → tests → patterns → pretrain → HNSW rebuild.
+        spawnWindowless('node', [resolve(__dirname, 'index-all.mjs')], 'sequential indexing chain');
+        // Neural patterns now loaded by moflo core routing — no external patching.
         break;
       }
 
@@ -469,55 +477,6 @@ function runIndexGuidanceBackground(specificFile = null) {
   spawnWindowless('node', indexArgs, desc);
 }
 
-// Run structural code map generator in background (non-blocking)
-function runCodeMapBackground() {
-  // Check auto_index.code_map flag in moflo.yaml (default: true)
-  const yamlPath = resolve(projectRoot, 'moflo.yaml');
-  if (existsSync(yamlPath)) {
-    try {
-      const content = readFileSync(yamlPath, 'utf-8');
-      const match = content.match(/auto_index:\s*\n(?:.*\n)*?\s+code_map:\s*(true|false)/);
-      if (match && match[1] === 'false') {
-        log('info', 'Code map generation disabled (auto_index.code_map: false)');
-        return;
-      }
-    } catch { /* ignore, proceed with indexing */ }
-  }
-
-  const codeMapScript = resolveBinOrLocal('flo-codemap', 'generate-code-map.mjs');
-
-  if (!codeMapScript) {
-    log('warn', 'Code map generator not found (checked npm bin + .claude/scripts/)');
-    return;
-  }
-
-  spawnWindowless('node', [codeMapScript], 'background code map generation');
-}
-
-// Run test file indexer in background (non-blocking)
-function runTestIndexBackground() {
-  // Check auto_index.tests flag in moflo.yaml (default: true)
-  const yamlPath = resolve(projectRoot, 'moflo.yaml');
-  if (existsSync(yamlPath)) {
-    try {
-      const content = readFileSync(yamlPath, 'utf-8');
-      const match = content.match(/auto_index:\s*\n(?:.*\n)*?\s+tests:\s*(true|false)/);
-      if (match && match[1] === 'false') {
-        log('info', 'Test indexing disabled (auto_index.tests: false)');
-        return;
-      }
-    } catch { /* ignore, proceed with indexing */ }
-  }
-
-  const testIndexScript = resolveBinOrLocal('flo-testmap', 'index-tests.mjs');
-
-  if (!testIndexScript) {
-    log('info', 'Test indexer not found (checked npm bin + .claude/scripts/)');
-    return;
-  }
-
-  spawnWindowless('node', [testIndexScript], 'background test indexing');
-}
 
 // Run ReasoningBank + MicroLoRA training + EWC++ consolidation in background (non-blocking)
 function runBackgroundTraining() {
@@ -625,29 +584,19 @@ function runDaemonStartBackground() {
   spawnWindowless('node', [localCli, 'daemon', 'start', '--quiet'], 'daemon');
 }
 
-// Run pretrain in background on session start (non-blocking)
-function runBackgroundPretrain() {
-  const localCli = getLocalCliPath();
-  if (!localCli) {
-    log('warn', 'Local CLI not found, skipping background pretrain');
-    return;
-  }
 
-  spawnWindowless('node', [localCli, 'hooks', 'pretrain'], 'background pretrain');
-}
-
-// Force HNSW rebuild in background to ensure all processes use identical fresh index
-// This fixes the issue where spawned agents return different search results than CLI/MCP
-function runHNSWRebuildBackground() {
+// Initialize embeddings ONNX engine on session start (non-blocking)
+function runEmbeddingsInitBackground() {
   const localCli = getLocalCliPath();
   if (!localCli) {
-    log('warn', 'Local CLI not found, skipping HNSW rebuild');
+    log('warn', 'Local CLI not found, skipping embeddings init');
     return;
   }
 
-  spawnWindowless('node', [localCli, 'memory', 'rebuild', '--force'], 'HNSW rebuild');
+  spawnWindowless('node', [localCli, 'embeddings', 'init'], 'embeddings init');
 }
 
+
 // Neural pattern application — now handled by moflo core routing (learned patterns
 // loaded from routing-outcomes.json by hooks-tools.ts getSemanticRouter).
 // No external patch script needed.

package/.claude/scripts/index-all.mjs

@@ -1,127 +1,193 @@
-#!/usr/bin/env node
-/**
- * Sequential indexer chain for session-start.
- *
- * Runs all DB-writing indexers one at a time to avoid sql.js last-write-wins
- * concurrency issues, then triggers HNSW rebuild once everything is committed.
- *
- * Spawned as a single detached background process by hooks.mjs session-start.
- */
-
-import { existsSync, appendFileSync } from 'fs';
-import { resolve, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { execFileSync } from 'child_process';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const projectRoot = resolve(__dirname, '../..');
-const LOG_PATH = resolve(projectRoot, '.claude/hooks.log');
-
-function log(msg) {
-  const ts = new Date().toISOString().replace('T', ' ').slice(0, 19);
-  const line = `[${ts}] [index-all] ${msg}\n`;
-  try { appendFileSync(LOG_PATH, line); } catch { /* ignore */ }
-}
-
-function resolveBin(binName, localScript) {
-  const mofloScript = resolve(projectRoot, 'node_modules/moflo/bin', localScript);
-  if (existsSync(mofloScript)) return mofloScript;
-  const npmBin = resolve(projectRoot, 'node_modules/.bin', binName);
-  if (existsSync(npmBin)) return npmBin;
-  const localPath = resolve(projectRoot, '.claude/scripts', localScript);
-  if (existsSync(localPath)) return localPath;
-  return null;
-}
-
-function getLocalCliPath() {
-  const paths = [
-    resolve(projectRoot, 'node_modules/moflo/src/@claude-flow/cli/bin/cli.js'),
-    resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
-    resolve(projectRoot, 'node_modules/.bin/flo'),
-    resolve(projectRoot, 'src/@claude-flow/cli/bin/cli.js'),
-  ];
-  for (const p of paths) {
-    if (existsSync(p)) return p;
-  }
-  return null;
-}
-
-function runStep(label, cmd, args, timeoutMs = 120_000) {
-  const start = Date.now();
-  log(`START ${label}`);
-  try {
-    execFileSync(cmd, args, {
-      cwd: projectRoot,
-      timeout: timeoutMs,
-      stdio: 'ignore',
-      windowsHide: true,
-    });
-    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-    log(`DONE  ${label} (${elapsed}s)`);
-    return true;
-  } catch (err) {
-    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-    log(`FAIL  ${label} (${elapsed}s): ${err.message?.split('\n')[0] || 'unknown'}`);
-    return false;
-  }
-}
-
-async function main() {
-  const startTime = Date.now();
-  log('Sequential indexing chain started');
-
-  // 1. Guidance indexer
-  const guidanceScript = resolveBin('flo-index', 'index-guidance.mjs');
-  if (guidanceScript) {
-    runStep('guidance-index', 'node', [guidanceScript]);
-  } else {
-    log('SKIP  guidance-index (script not found)');
-  }
-
-  // 2. Code map generator (the big one — ~22s)
-  const codeMapScript = resolveBin('flo-codemap', 'generate-code-map.mjs');
-  if (codeMapScript) {
-    runStep('code-map', 'node', [codeMapScript], 180_000);
-  } else {
-    log('SKIP  code-map (script not found)');
-  }
-
-  // 3. Test indexer
-  const testScript = resolveBin('flo-testmap', 'index-tests.mjs');
-  if (testScript) {
-    runStep('test-index', 'node', [testScript]);
-  } else {
-    log('SKIP  test-index (script not found)');
-  }
-
-  // 4. Patterns indexer
-  const patternsScript = resolveBin('flo-patterns', 'index-patterns.mjs');
-  if (patternsScript) {
-    runStep('patterns-index', 'node', [patternsScript]);
-  } else {
-    log('SKIP  patterns-index (script not found)');
-  }
-
-  // 5. Pretrain (extracts patterns from repository)
-  const localCli = getLocalCliPath();
-  if (localCli) {
-    runStep('pretrain', 'node', [localCli, 'hooks', 'pretrain']);
-  } else {
-    log('SKIP  pretrain (CLI not found)');
-  }
-
-  // 6. HNSW rebuild — MUST run last, after all writes are committed (#81)
-  if (localCli) {
-    runStep('hnsw-rebuild', 'node', [localCli, 'memory', 'rebuild', '--force']);
-  } else {
-    log('SKIP  hnsw-rebuild (CLI not found)');
-  }
-
-  const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-  log(`Sequential indexing chain complete (${totalElapsed}s)`);
-}
-
-main().catch(err => {
-  log(`FATAL: ${err.message}`);
-  process.exit(1);
-});
+#!/usr/bin/env node
+/**
+ * Sequential indexer chain for session-start.
+ *
+ * Runs all DB-writing indexers one at a time to avoid sql.js last-write-wins
+ * concurrency issues (#78), then triggers HNSW rebuild once everything is
+ * committed (#81).
+ *
+ * Spawned as a single detached background process by hooks.mjs session-start.
+ */
+
+import { existsSync, appendFileSync, readFileSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { execFileSync } from 'child_process';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Detect project root by walking up from cwd to find package.json.
+// IMPORTANT: Do NOT use resolve(__dirname, '..') — this script lives in bin/
+// during development but gets synced to .claude/scripts/ in consumer projects,
+// so __dirname-relative paths break. findProjectRoot() works in both locations.
+function findProjectRoot() {
+  let dir = process.cwd();
+  const root = resolve(dir, '/');
+  while (dir !== root) {
+    if (existsSync(resolve(dir, 'package.json'))) return dir;
+    dir = dirname(dir);
+  }
+  return process.cwd();
+}
+
+const projectRoot = findProjectRoot();
+const LOG_PATH = resolve(projectRoot, '.swarm/hooks.log');
+
+function log(msg) {
+  const ts = new Date().toISOString().replace('T', ' ').slice(0, 19);
+  const line = `[${ts}] [index-all] ${msg}\n`;
+  try { appendFileSync(LOG_PATH, line); } catch { /* ignore */ }
+}
+
+function resolveBin(binName, localScript) {
+  const mofloScript = resolve(projectRoot, 'node_modules/moflo/bin', localScript);
+  if (existsSync(mofloScript)) return mofloScript;
+  const npmBin = resolve(projectRoot, 'node_modules/.bin', binName);
+  if (existsSync(npmBin)) return npmBin;
+  const localPath = resolve(projectRoot, '.claude/scripts', localScript);
+  if (existsSync(localPath)) return localPath;
+  // Also check bin/ directory (for development use)
+  const binPath = resolve(projectRoot, 'bin', localScript);
+  if (existsSync(binPath)) return binPath;
+  return null;
+}
+
+function getLocalCliPath() {
+  const paths = [
+    resolve(projectRoot, 'node_modules/moflo/src/@claude-flow/cli/bin/cli.js'),
+    resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
+    resolve(projectRoot, 'node_modules/.bin/flo'),
+    // Development: local CLI
+    resolve(projectRoot, 'src/@claude-flow/cli/bin/cli.js'),
+  ];
+  for (const p of paths) {
+    if (existsSync(p)) return p;
+  }
+  return null;
+}
+
+/** Read moflo.yaml once and cache auto_index flags. */
+let _autoIndexFlags = null;
+function isIndexEnabled(key) {
+  if (_autoIndexFlags === null) {
+    _autoIndexFlags = {};
+    const yamlPath = resolve(projectRoot, 'moflo.yaml');
+    if (existsSync(yamlPath)) {
+      try {
+        const content = readFileSync(yamlPath, 'utf-8');
+        for (const k of ['guidance', 'code_map', 'tests', 'patterns']) {
+          const re = new RegExp(`auto_index:\\s*\\n(?:.*\\n)*?\\s+${k}:\\s*(true|false)`);
+          const match = content.match(re);
+          _autoIndexFlags[k] = match ? match[1] !== 'false' : true;
+        }
+      } catch { /* ignore, all default to true */ }
+    }
+  }
+  return _autoIndexFlags[key] !== false;
+}
+
+function runStep(label, cmd, args, timeoutMs = 120_000) {
+  const start = Date.now();
+  log(`START ${label}`);
+  try {
+    execFileSync(cmd, args, {
+      cwd: projectRoot,
+      timeout: timeoutMs,
+      stdio: 'ignore',
+      windowsHide: true,
+    });
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+    log(`DONE  ${label} (${elapsed}s)`);
+    return true;
+  } catch (err) {
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+    log(`FAIL  ${label} (${elapsed}s): ${err.message?.split('\n')[0] || 'unknown'}`);
+    return false;
+  }
+}
+
+async function main() {
+  const startTime = Date.now();
+  log('Sequential indexing chain started');
+
+  // 1. Guidance indexer
+  if (isIndexEnabled('guidance')) {
+    const guidanceScript = resolveBin('flo-index', 'index-guidance.mjs');
+    if (guidanceScript) {
+      runStep('guidance-index', 'node', [guidanceScript, '--no-embeddings']);
+    } else {
+      log('SKIP  guidance-index (script not found)');
+    }
+  } else {
+    log('SKIP  guidance-index (disabled in moflo.yaml)');
+  }
+
+  // 2. Code map generator (the big one — ~22s)
+  if (isIndexEnabled('code_map')) {
+    const codeMapScript = resolveBin('flo-codemap', 'generate-code-map.mjs');
+    if (codeMapScript) {
+      runStep('code-map', 'node', [codeMapScript, '--no-embeddings'], 180_000);
+    } else {
+      log('SKIP  code-map (script not found)');
+    }
+  } else {
+    log('SKIP  code-map (disabled in moflo.yaml)');
+  }
+
+  // 3. Test indexer
+  if (isIndexEnabled('tests')) {
+    const testScript = resolveBin('flo-testmap', 'index-tests.mjs');
+    if (testScript) {
+      runStep('test-index', 'node', [testScript, '--no-embeddings']);
+    } else {
+      log('SKIP  test-index (script not found)');
+    }
+  } else {
+    log('SKIP  test-index (disabled in moflo.yaml)');
+  }
+
+  // 4. Patterns indexer
+  if (isIndexEnabled('patterns')) {
+    const patternsScript = resolveBin('flo-patterns', 'index-patterns.mjs');
+    if (patternsScript) {
+      runStep('patterns-index', 'node', [patternsScript]);
+    } else {
+      log('SKIP  patterns-index (script not found)');
+    }
+  } else {
+    log('SKIP  patterns-index (disabled in moflo.yaml)');
+  }
+
+  // 5. Pretrain (extracts patterns from repository)
+  const localCli = getLocalCliPath();
+  if (localCli) {
+    runStep('pretrain', 'node', [localCli, 'hooks', 'pretrain']);
+  } else {
+    log('SKIP  pretrain (CLI not found)');
+  }
+
+  // 6. Build embeddings — single pass for ALL namespaces, after all indexers finish.
+  //    Individual indexers are called with --no-embeddings to prevent background
+  //    embedding spawns that race with this chain (sql.js last-write-wins).
+  const embeddingsScript = resolveBin('flo-embeddings', 'build-embeddings.mjs');
+  if (embeddingsScript) {
+    runStep('build-embeddings', 'node', [embeddingsScript], 300_000);
+  } else {
+    log('SKIP  build-embeddings (script not found)');
+  }
+
+  // 7. HNSW rebuild — MUST run last, after all writes are committed (#81)
+  if (localCli) {
+    runStep('hnsw-rebuild', 'node', [localCli, 'memory', 'rebuild', '--force']);
+  } else {
+    log('SKIP  hnsw-rebuild (CLI not found)');
+  }
+
+  const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  log(`Sequential indexing chain complete (${totalElapsed}s)`);
+}
+
+main().catch(err => {
+  log(`FATAL: ${err.message}`);
+  process.exit(1);
+});

package/.claude/scripts/index-guidance.mjs

@@ -106,7 +106,18 @@ function loadGuidanceDirs() {
     ? bundledShippedDir
     : resolve(mofloRoot, '.claude/guidance');
   const projectGuidanceDir = resolve(projectRoot, '.claude/guidance');
+
+  // Detect self-reference: if the project IS the moflo package, skip bundled scan
+  // to avoid double-indexing the same files under two prefixes.
+  let isSelfRef = false;
+  try {
+    const projPkg = JSON.parse(readFileSync(resolve(projectRoot, 'package.json'), 'utf-8'));
+    const mofloPkg = JSON.parse(readFileSync(resolve(mofloRoot, 'package.json'), 'utf-8'));
+    isSelfRef = projPkg.name === mofloPkg.name;
+  } catch { /* ignore — defaults to false */ }
+
   if (
+    !isSelfRef &&
     existsSync(bundledGuidanceDir) &&
     resolve(bundledGuidanceDir) !== resolve(projectGuidanceDir) &&
     resolve(bundledGuidanceDir) !== resolve(projectGuidanceDir, 'shipped')
@@ -819,8 +830,17 @@ if (!specificFile) {
   }
 }
 
-// Write changes back to disk and close
+// Write changes back to disk
 if (docsIndexed > 0 || chunksIndexed > 0 || staleRemoved > 0) saveDb(db);
+
+// Check for entries missing embeddings (e.g. prior background run failed)
+let missingEmbeddings = 0;
+{
+  const stmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE namespace = ? AND (embedding IS NULL OR embedding = '')`);
+  stmt.bind([NAMESPACE]);
+  if (stmt.step()) missingEmbeddings = stmt.getAsObject().cnt;
+  stmt.free();
+}
 db.close();
 
 console.log('');
@@ -840,9 +860,13 @@ log(`    • Hierarchical links (h2 -> h3 parent/children)`);
 log(`    • Context overlap: ${overlapPercent}% (contextBefore/contextAfter)`);
 log('═══════════════════════════════════════════════════════════');
 
-// Generate embeddings for new entries (unless skipped or nothing changed)
+// Generate embeddings for new entries or backfill missing ones
 // Runs in BACKGROUND to avoid blocking startup
-if (!skipEmbeddings && (docsIndexed > 0 || chunksIndexed > 0)) {
+const needsEmbeddings = (docsIndexed > 0 || chunksIndexed > 0 || missingEmbeddings > 0);
+if (!skipEmbeddings && needsEmbeddings) {
+  if (missingEmbeddings > 0 && docsIndexed === 0 && chunksIndexed === 0) {
+    log(`${missingEmbeddings} entries missing embeddings — backfilling...`);
+  }
   console.log('');
   log('Spawning embedding generation in background...');