moflo 4.9.6 → 4.9.8

package/bin/index-all.mjs

@@ -2,9 +2,13 @@
 /**
  * Sequential indexer chain for session-start.
  *
- * Runs all DB-writing indexers one at a time to avoid sql.js last-write-wins
- * concurrency issues (#78), then triggers HNSW rebuild once everything is
- * committed (#81).
+ * Each step is gated independently — see `lib/index-fingerprint.mjs`. The
+ * orchestrator just walks the plan, asks the gate per step, runs the
+ * survivors, and saves the post-run fingerprint when each succeeds.
+ *
+ * Steps run sequentially (DB-writing) to avoid sql.js last-write-wins
+ * concurrency issues (#78). HNSW rebuild is last, after every other step
+ * has committed (#81).
  *
  * Spawned as a single detached background process by hooks.mjs session-start.
  */
@@ -15,6 +19,21 @@ import { fileURLToPath } from 'url';
 import { spawn, spawnSync } from 'child_process';
 import { platform } from 'os';
 import { hnswIndexPath } from './lib/moflo-paths.mjs';
+import {
+  decideStepGate,
+  computeStepFingerprint,
+  saveStepFingerprint,
+  cleanupLegacyFingerprint,
+} from './lib/index-fingerprint.mjs';
+
+// Cap fastembed/ONNX thread count when spawning the heavy steps. Without
+// this, ONNX defaults to one thread per CPU core (22+ on a modern dev box),
+// pegging the entire machine while the indexer runs. 2 threads keeps
+// re-embedding throughput acceptable while leaving the box usable.
+const ONNX_THREAD_CAP = {
+  OMP_NUM_THREADS: '2',
+  ONNXRUNTIME_INTRA_OP_NUM_THREADS: '2',
+};
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -56,7 +75,6 @@ function resolveBin(binName, localScript) {
 
 function getLocalCliPath() {
   const paths = [
-    resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
     resolve(projectRoot, 'node_modules/moflo/bin/cli.js'),
     resolve(projectRoot, 'node_modules/.bin/flo'),
     // Development: local CLI
@@ -110,7 +128,7 @@ function killProcessTree(child) {
   }
 }
 
-function runStep(label, cmd, args, timeoutMs = 120_000) {
+function runStep(label, cmd, args, timeoutMs = 120_000, extraEnv = null) {
   return new Promise((resolveStep) => {
     const start = Date.now();
     log(`START ${label}`);
@@ -119,6 +137,7 @@ function runStep(label, cmd, args, timeoutMs = 120_000) {
       stdio: 'ignore',
       windowsHide: true,
       detached: platform() !== 'win32', // POSIX: own process group for tree-kill
+      env: extraEnv ? { ...process.env, ...extraEnv } : process.env,
     });
     let timedOut = false;
     const timer = setTimeout(() => {
@@ -148,101 +167,140 @@ function runStep(label, cmd, args, timeoutMs = 120_000) {
   });
 }
 
-async function main() {
-  const startTime = Date.now();
-  log('Sequential indexing chain started');
-
-  // 1. Guidance indexer
-  if (isIndexEnabled('guidance')) {
-    const guidanceScript = resolveBin('flo-index', 'index-guidance.mjs');
-    if (guidanceScript) {
-      await runStep('guidance-index', 'node', [guidanceScript, '--no-embeddings']);
-    } else {
-      log('SKIP  guidance-index (script not found)');
-    }
-  } else {
-    log('SKIP  guidance-index (disabled in moflo.yaml)');
-  }
+/**
+ * Build the ordered step plan. Each entry is `{ name, cmd, args, timeoutMs, env? }`.
+ * Steps disabled in moflo.yaml or whose script can't be located are filtered
+ * out here so the run loop only sees runnable steps.
+ */
+function buildStepPlan() {
+  const plan = [];
+  const localCli = getLocalCliPath();
 
-  // 2. Code map generator (the big one — ~22s)
-  if (isIndexEnabled('code_map')) {
-    const codeMapScript = resolveBin('flo-codemap', 'generate-code-map.mjs');
-    if (codeMapScript) {
-      await runStep('code-map', 'node', [codeMapScript, '--no-embeddings'], 180_000);
-    } else {
-      log('SKIP  code-map (script not found)');
+  const consider = (name, cfgKey, scriptName, binName, args, timeoutMs = 120_000, env = null) => {
+    if (cfgKey && !isIndexEnabled(cfgKey)) {
+      log(`SKIP  ${name} (disabled in moflo.yaml)`);
+      return;
     }
-  } else {
-    log('SKIP  code-map (disabled in moflo.yaml)');
-  }
-
-  // 3. Test indexer
-  if (isIndexEnabled('tests')) {
-    const testScript = resolveBin('flo-testmap', 'index-tests.mjs');
-    if (testScript) {
-      await runStep('test-index', 'node', [testScript, '--no-embeddings']);
-    } else {
-      log('SKIP  test-index (script not found)');
+    const script = scriptName ? resolveBin(binName, scriptName) : null;
+    if (scriptName && !script) {
+      log(`SKIP  ${name} (script not found)`);
+      return;
     }
-  } else {
-    log('SKIP  test-index (disabled in moflo.yaml)');
-  }
+    plan.push({
+      name,
+      cmd: 'node',
+      args: scriptName ? [script, ...args] : args,
+      timeoutMs,
+      env,
+    });
+  };
 
-  // 4. Patterns indexer
-  if (isIndexEnabled('patterns')) {
-    const patternsScript = resolveBin('flo-patterns', 'index-patterns.mjs');
-    if (patternsScript) {
-      await runStep('patterns-index', 'node', [patternsScript]);
-    } else {
-      log('SKIP  patterns-index (script not found)');
-    }
-  } else {
-    log('SKIP  patterns-index (disabled in moflo.yaml)');
-  }
+  consider('guidance-index', 'guidance', 'index-guidance.mjs', 'flo-index',   ['--no-embeddings']);
+  consider('code-map',       'code_map', 'generate-code-map.mjs', 'flo-codemap', ['--no-embeddings'], 180_000);
+  consider('test-index',     'tests',    'index-tests.mjs',    'flo-testmap', ['--no-embeddings']);
+  consider('patterns-index', 'patterns', 'index-patterns.mjs', 'flo-patterns', []);
 
-  // 5. Pretrain (extracts patterns from repository)
-  const localCli = getLocalCliPath();
+  // Pretrain extracts patterns from the repo via the CLI subcommand. No
+  // direct script — invoke through the local flo binary.
   if (localCli) {
-    await runStep('pretrain', 'node', [localCli, 'hooks', 'pretrain']);
+    plan.push({
+      name: 'pretrain',
+      cmd: 'node',
+      args: [localCli, 'hooks', 'pretrain'],
+      timeoutMs: 120_000,
+    });
   } else {
     log('SKIP  pretrain (CLI not found)');
   }
 
-  // 6. Build embeddings — single pass for ALL namespaces, after all indexers finish.
-  //    Individual indexers are called with --no-embeddings to prevent background
-  //    embedding spawns that race with this chain (sql.js last-write-wins).
-  const embeddingsScript = resolveBin('flo-embeddings', 'build-embeddings.mjs');
-  if (embeddingsScript) {
-    await runStep('build-embeddings', 'node', [embeddingsScript], 300_000);
+  // build-embeddings runs fastembed → thread-capped to keep CPU usable.
+  consider('build-embeddings', null, 'build-embeddings.mjs', 'flo-embeddings', [], 300_000, ONNX_THREAD_CAP);
+
+  // HNSW MUST run last (after all DB writes are committed, #81). Same thread
+  // cap — rebuild-index loads fastembed for stats lookups.
+  //
+  // No `--force`: the embeddings-migration service (run by the launcher
+  // before this chain) handles model bumps, and `build-embeddings` above
+  // fills any rows that lack embeddings. So `rebuild-index` finds nothing
+  // to embed in steady state and takes the no-work path, which still
+  // refreshes the HNSW sidecar via `writeSidecarOrFail` and is followed by
+  // the existsSync post-check below. `--force` only added a 4000-row
+  // re-embed that the fingerprint gate (#858) is specifically trying to
+  // avoid (#859).
+  if (localCli) {
+    plan.push({
+      name: 'hnsw-rebuild',
+      cmd: 'node',
+      args: [localCli, 'memory', 'rebuild-index'],
+      timeoutMs: 300_000,
+      env: ONNX_THREAD_CAP,
+    });
   } else {
-    log('SKIP  build-embeddings (script not found)');
+    log('SKIP  hnsw-rebuild (CLI not found)');
   }
 
-  // 7. HNSW rebuild — MUST run last, after all writes are committed (#81).
-  //    rebuild-index now also writes the binary HNSW sidecar at
-  //    .moflo/hnsw.index, which can take longer than the default 120s on a
-  //    populated consumer DB — match build-embeddings' 300s budget.
+  return plan;
+}
+
+async function main() {
+  const startTime = Date.now();
+  const plan = buildStepPlan();
+
+  let ranAny = false;
+  let hnswAttempted = false;
   let hnswOk = true;
-  if (localCli) {
-    const ok = await runStep('hnsw-rebuild', 'node', [localCli, 'memory', 'rebuild-index', '--force'], 300_000);
+
+  for (const step of plan) {
+    const gate = decideStepGate(step.name, projectRoot);
+    if (gate.skip) {
+      log(`SKIP  ${step.name} (${gate.reason})`);
+      continue;
+    }
+    log(`RUN   ${step.name} (${gate.reason})`);
+    if (step.name === 'hnsw-rebuild') hnswAttempted = true;
+    const ok = await runStep(step.name, step.cmd, step.args, step.timeoutMs, step.env || null);
     if (ok) {
-      const sidecar = hnswIndexPath(projectRoot);
-      if (!existsSync(sidecar)) {
-        // Loud failure: missing sidecar means cold-start memory search
-        // silently rebuilds from SQL on every consumer process — the exact
-        // regression this guard exists to surface.
-        log(`FAIL  hnsw-rebuild post-check: sidecar missing at ${sidecar}`);
-        hnswOk = false;
+      // POST-run fingerprint: re-compute to capture any state mutated by
+      // this step (e.g. build-embeddings bumping memory.db mtime). Saving
+      // the POST value lets next session correctly compare against the
+      // stable post-step state.
+      try {
+        const post = computeStepFingerprint(step.name, projectRoot);
+        if (!saveStepFingerprint(step.name, projectRoot, post)) {
+          log(`WARN  ${step.name} fingerprint save failed (next session will re-run)`);
+        }
+      } catch (err) {
+        const msg = (err && err.message ? err.message.split('\n')[0] : 'unknown');
+        log(`WARN  ${step.name} fingerprint compute failed: ${msg}`);
       }
-    } else {
+      ranAny = true;
+    } else if (step.name === 'hnsw-rebuild') {
       hnswOk = false;
     }
-  } else {
-    log('SKIP  hnsw-rebuild (CLI not found)');
   }
 
+  // hnsw-rebuild post-check: sidecar must physically exist after the step
+  // ran successfully. Missing sidecar means cold-start memory search will
+  // silently rebuild from SQL on every consumer process — the regression
+  // this guard exists to surface (#854). Only meaningful when we actually
+  // tried to rebuild.
+  if (hnswAttempted && hnswOk) {
+    const sidecar = hnswIndexPath(projectRoot);
+    if (!existsSync(sidecar)) {
+      log(`FAIL  hnsw-rebuild post-check: sidecar missing at ${sidecar}`);
+      hnswOk = false;
+    }
+  }
+
+  // Always tidy up the v1 fingerprint file from 4.9.7 — even on all-skip
+  // sessions, otherwise the orphan survives indefinitely.
+  cleanupLegacyFingerprint(projectRoot);
+
   const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-  log(`Sequential indexing chain complete (${totalElapsed}s)`);
+  log(ranAny
+    ? `Sequential indexing chain complete (${totalElapsed}s)`
+    : `Sequential indexing chain skipped — all steps gated unchanged (${totalElapsed}s)`);
+
   if (!hnswOk) process.exit(1);
 }
 

package/bin/session-start-launcher.mjs

@@ -14,6 +14,17 @@ import { fileURLToPath } from 'url';
 import { mofloDir } from './lib/moflo-paths.mjs';
 import { repairMemoryDbIfCorrupt } from './lib/db-repair.mjs';
 
+// Headless skip (#860). The daemon's headless workers spawn `claude --print`
+// with CLAUDE_CODE_HEADLESS=true (see src/cli/services/headless-worker-
+// executor.ts). Each spawned Claude inherits SessionStart hooks, which
+// would re-enter this launcher and fork the indexer chain — bumping
+// memory.db mtime, invalidating the 4.9.7 fingerprint gate, and pegging
+// CPU on the daemon's 15-min worker cycle.
+if (process.env.CLAUDE_CODE_HEADLESS === 'true' || process.env.CLAUDE_CODE_HEADLESS === '1') {
+  emitWarning('session-start-launcher skipped (CLAUDE_CODE_HEADLESS=true)');
+  process.exit(0);
+}
+
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 // Single source of truth for the launcher's guidance-mirror header. Section 3

package/dist/src/cli/version.js

@@ -2,5 +2,5 @@
  * Auto-generated by build. Do not edit manually.
  * Source of truth: root package.json → scripts/sync-version.mjs
  */
-export const VERSION = '4.9.6';
+export const VERSION = '4.9.8';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "moflo",
-  "version": "4.9.6",
+  "version": "4.9.8",
   "description": "MoFlo — AI agent orchestration for Claude Code. A standalone, opinionated toolkit with semantic memory, learned routing, gates, spells, and the /flo issue-execution skill.",
   "main": "dist/src/cli/index.js",
   "type": "module",
@@ -81,7 +81,7 @@
     "@typescript-eslint/eslint-plugin": "^7.18.0",
     "@typescript-eslint/parser": "^7.18.0",
     "eslint": "^8.0.0",
-    "moflo": "^4.9.5",
+    "moflo": "^4.9.7",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
     "vitest": "^4.0.0"