moflo 4.9.1 → 4.9.3

package/.claude/guidance/shipped/moflo-error-handling.md

@@ -43,6 +43,31 @@
 
 ---
 
+## Transient Errors Must Use Retry + Circuit Breaker
+
+**Wrap every transient-failure-capable operation in a retry helper with exponential backoff and a circuit breaker.** One-shot try-and-log on a transient class strands users in partial-state loops (#854: Windows file-lock + AV scan race left stale `.claude/helpers/gate.cjs` across 8+ moflo bumps).
+
+| Element | Default |
+|---------|---------|
+| Retryable codes | `EBUSY`, `EPERM`, `EACCES`, `EAGAIN`, `ETIMEDOUT`, `ECONNRESET`; HTTP 5xx/429/408 |
+| Hard codes (no retry) | `ENOENT`, `EISDIR`, `EEXIST`, validation errors |
+| Backoff (filesystem) | `[50, 200, 800]ms` — 3 retries |
+| Circuit breaker | Open after 5 distinct failures; tail runs with `maxAttempts=1` |
+| Exhaustion handling | Log error AND name the healer (e.g. `run 'flo doctor --fix' to repair`) |
+
+**Reference implementation:** `syncWithRetry` in `bin/session-start-launcher.mjs`. Use it; do not invent ad-hoc retries.
+
+```js
+// FORBIDDEN
+try { copyFileSync(src, dest); } catch (err) { console.warn(err.message); }
+
+// CORRECT
+const result = await syncWithRetry(() => copyFileSync(src, dest));
+if (!result.ok) syncFailures.push({ key, message: `${errMessage(result.err)} (retried after ${result.code})` });
+```
+
+---
+
 ## See Also
 
 - `.claude/guidance/shipped/moflo-source-hygiene.md` — General source code standards

package/.claude/guidance/shipped/moflo-source-hygiene.md

@@ -68,6 +68,22 @@ Compiled output is generated by `npm run build` (single `tsc` run from repo root
 
 ---
 
+## Async-Eligible Operations Must Be Async By Default
+
+**Use the async API for any operation that has one, unless a documented reason forces sync.** Sync IO and busy-waits block the event loop — on a hot path that pins a CPU core, stalls timers, and turns a "best-effort" failure into a hang. Issue #854 burned 5s of CPU to a `while (Date.now() < end)` busy-wait inside an already-async launcher.
+
+| Operation | Use | Avoid |
+|-----------|-----|-------|
+| File IO inside an `async` function | `fs/promises` (`readFile`, `copyFile`, `mkdir`) | `*Sync` variants |
+| Delays | `await new Promise(r => setTimeout(r, ms))` | `while (Date.now() < end)` busy-wait |
+| Spawning long-running children | `await new Promise` around `spawn`/`execFile` | `execFileSync`/`spawnSync` (Windows timeouts orphan children — #744) |
+
+**Sync is OK only when:** the function cannot be `async` (top-level config reader before any await; certain CommonJS hooks); the IO is microscopic on a known-fast path (one `existsSync` probe, one tiny `readFileSync`); or conversion churn outweighs the benefit. Document the reason in a one-line comment.
+
+**Rule of thumb:** if anything in your call stack already `await`s, every IO/delay below it must be async too — mixing sync IO inside an async function is a smell.
+
+---
+
 ## When Creating New Files
 
 Before creating any new source file, verify:

package/bin/build-embeddings.mjs

@@ -203,6 +203,29 @@ function writeVectorStatsCache(stats, nsStats) {
 // Main
 // ============================================================================
 
+// Build (or skip) the HNSW sidecar — shared between the all-embedded fast path
+// and the post-embedding finalize path. Issue #854: the early-return path used
+// to skip this entirely, leaving consumers with `hasHnsw: false` permanently
+// after the embeddings migration deleted the stale sidecar without rebuilding.
+// `stats` is passed in by both call sites so we don't redundantly run the
+// COUNT aggregate the caller already executed. `alwaysRebuild` is set by the
+// post-embedding path because newly-embedded rows always invalidate the
+// existing sidecar.
+async function ensureHnswSidecar(stats, { alwaysRebuild = false } = {}) {
+  if (stats.withEmbeddings <= 0) return false;
+  const sidecarExists = existsSync(hnswIndexPath(projectRoot));
+  if (sidecarExists && !force && !alwaysRebuild) return false;
+  log(sidecarExists ? 'Rebuilding HNSW sidecar...' : 'Building HNSW sidecar...');
+  const result = await buildAndWriteHnswSidecar(DB_PATH, projectRoot, {
+    dimensions: EMBEDDING_DIMS,
+  });
+  log(`HNSW sidecar: ${result.vectorCount} vectors → ${result.sidecarPath} (${(result.bytes / 1024).toFixed(1)} KB)`);
+  if (!existsSync(result.sidecarPath)) {
+    throw new Error(`HNSW sidecar missing after write: ${result.sidecarPath}`);
+  }
+  return true;
+}
+
 async function main() {
   console.log('');
   log('═══════════════════════════════════════════════════════════');
@@ -217,6 +240,8 @@ async function main() {
     log('All entries already have embeddings');
     const stats = getEmbeddingStats(db);
     log(`Total: ${stats.withEmbeddings}/${stats.total} entries embedded`);
+    // HNSW first so vector-stats reports the post-rebuild `hasHnsw` (#854).
+    await ensureHnswSidecar(stats);
     writeVectorStatsCache(stats, getNamespaceStats(db));
     db.close();
     return;
@@ -312,22 +337,14 @@ async function main() {
   }
   log('═══════════════════════════════════════════════════════════');
 
+  // Rebuild the HNSW sidecar before vector-stats so `hasHnsw` reflects the
+  // post-rebuild state in the same session (#854). Newly-embedded rows
+  // invalidate the existing sidecar, so we force the rebuild here even
+  // when the sidecar already exists.
+  await ensureHnswSidecar(stats, { alwaysRebuild: true });
+
   writeVectorStatsCache(stats, nsStats);
   db.close();
-
-  // Rebuild + persist the HNSW sidecar so cold-start memory searches skip
-  // the SQL→graph rebuild. Failure here exits non-zero — index-all.mjs and
-  // any caller of this script depend on the sidecar landing on disk.
-  if (stats.withEmbeddings > 0) {
-    log('Building HNSW sidecar...');
-    const result = await buildAndWriteHnswSidecar(DB_PATH, projectRoot, {
-      dimensions: EMBEDDING_DIMS,
-    });
-    log(`HNSW sidecar: ${result.vectorCount} vectors → ${result.sidecarPath} (${(result.bytes / 1024).toFixed(1)} KB)`);
-    if (!existsSync(result.sidecarPath)) {
-      throw new Error(`HNSW sidecar missing after write: ${result.sidecarPath}`);
-    }
-  }
 }
 
 main().catch(err => {

package/bin/lib/moflo-paths.mjs

@@ -1,29 +1,16 @@
 /**
- * Pure-JS counterpart to src/cli/services/moflo-paths.ts (#699, #727).
+ * Pure-JS counterpart to src/cli/services/moflo-paths.ts.
  *
  * Lives in bin/lib because session-start-launcher.mjs and other bin/ scripts
  * run before any TS compilation has happened — they can't import the .ts
  * source. The TS version is the canonical programmatic API; this version
- * mirrors the same algorithm so migration also runs from the consumer
- * launcher path. Algorithm parity is enforced by the parity case in
- * src/cli/__tests__/services/moflo-paths-migration.test.ts.
+ * exposes the same path constants + helpers.
+ *
+ * Per #851, the legacy `.claude-flow/` rename + `.swarm/memory.db` byte-copy
+ * helpers no longer ship: the version-bump-gated cherry-pick lives entirely
+ * in the launcher and the TS service `cli/services/cherry-pick-learnings.ts`.
  */
-import {
-  closeSync,
-  copyFileSync,
-  existsSync,
-  mkdirSync,
-  openSync,
-  readFileSync,
-  readSync,
-  readdirSync,
-  renameSync,
-  rmdirSync,
-  statSync,
-  unlinkSync,
-  writeFileSync,
-} from 'node:fs';
-import { dirname, join } from 'node:path';
+import { join } from 'node:path';
 
 export const MOFLO_DIR = '.moflo';
 export const MEMORY_DB_FILE = 'moflo.db';
@@ -70,199 +57,3 @@ export function memoryDbCandidatePaths(projectRoot) {
     join(projectRoot, '.claude', LEGACY_MEMORY_DB_FILE),
   ];
 }
-
-/**
- * One-time migration of `.claude-flow/` → `.moflo/`. Idempotent — safe to call
- * on every session start. See moflo-paths.ts for the full contract.
- *
- * Returns `{ migrated, reason?, movedCount?, collisions? }`. The launcher
- * uses `movedCount` for the "migrated N files" message and `collisions` to
- * warn about subdirs (e.g. models/) that exist in both locations.
- */
-export function migrateClaudeFlowToMoflo(projectRoot) {
-  const legacy = legacyClaudeFlowDir(projectRoot);
-  const target = mofloDir(projectRoot);
-
-  if (!existsSync(legacy)) return { migrated: false, reason: 'no-legacy' };
-
-  if (!existsSync(target)) {
-    let movedCount = 0;
-    try { movedCount = readdirSync(legacy).length; } catch { /* count is cosmetic */ }
-    renameSync(legacy, target);
-    rewriteEmbeddingsModelPath(projectRoot);
-    return { migrated: true, movedCount };
-  }
-
-  let entries;
-  try {
-    entries = readdirSync(legacy);
-  } catch {
-    return { migrated: false, reason: 'legacy-unreadable' };
-  }
-
-  let moved = 0;
-  let modelsMoved = false;
-  const collisions = [];
-  for (const name of entries) {
-    const dst = join(target, name);
-    if (existsSync(dst)) {
-      collisions.push(name);
-      continue;
-    }
-    try {
-      renameSync(join(legacy, name), dst);
-      moved++;
-      if (name === 'models') modelsMoved = true;
-    } catch {
-      // Best-effort — single failed move shouldn't abort the rest.
-    }
-  }
-
-  try {
-    if (readdirSync(legacy).length === 0) rmdirSync(legacy);
-  } catch {
-    // Non-fatal — leftover legacy dir means migration runs next time.
-  }
-
-  if (modelsMoved) rewriteEmbeddingsModelPath(projectRoot);
-
-  if (moved === 0) {
-    return { migrated: false, reason: 'merged-nothing', movedCount: 0, collisions };
-  }
-  return { migrated: true, movedCount: moved, collisions };
-}
-
-/**
- * Rewrite `.moflo/embeddings.json:modelPath` if it still references the
- * legacy `.claude-flow/` location (#735). Best-effort: file-not-present,
- * malformed JSON, missing field, or already-correct path → silent no-op.
- * Mirrors the TS twin in src/cli/services/moflo-paths.ts. Uses tmp+rename
- * so SIGINT mid-flush can't leave embeddings.json truncated.
- */
-export function rewriteEmbeddingsModelPath(projectRoot) {
-  const cfgPath = join(projectRoot, MOFLO_DIR, 'embeddings.json');
-
-  let raw;
-  try { raw = readFileSync(cfgPath, 'utf8'); } catch { return false; }
-
-  let cfg;
-  try { cfg = JSON.parse(raw); } catch { return false; }
-
-  if (typeof cfg.modelPath !== 'string') return false;
-  if (!cfg.modelPath.includes(LEGACY_CLAUDE_FLOW_DIR)) return false;
-
-  cfg.modelPath = cfg.modelPath.split(LEGACY_CLAUDE_FLOW_DIR).join(MOFLO_DIR);
-
-  const tmpPath = `${cfgPath}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 8)}`;
-  try {
-    writeFileSync(tmpPath, JSON.stringify(cfg, null, 2));
-    renameSync(tmpPath, cfgPath);
-    return true;
-  } catch {
-    try { unlinkSync(tmpPath); } catch { /* best-effort cleanup */ }
-    return false;
-  }
-}
-
-const SQLITE_MAGIC_HEADER = Buffer.from('SQLite format 3\0', 'utf8');
-
-function looksLikeSqliteFile(filePath) {
-  let fd = null;
-  try {
-    fd = openSync(filePath, 'r');
-    const buf = Buffer.alloc(SQLITE_MAGIC_HEADER.length);
-    const read = readSync(fd, buf, 0, buf.length, 0);
-    if (read < SQLITE_MAGIC_HEADER.length) return false;
-    return buf.equals(SQLITE_MAGIC_HEADER);
-  } catch {
-    return false;
-  } finally {
-    if (fd !== null) try { closeSync(fd); } catch { /* non-fatal */ }
-  }
-}
-
-function verifyByteEqual(srcPath, dstPath) {
-  try {
-    const srcStat = statSync(srcPath);
-    const dstStat = statSync(dstPath);
-    if (srcStat.size !== dstStat.size) return false;
-    const srcBuf = readFileSync(srcPath);
-    const dstBuf = readFileSync(dstPath);
-    return srcBuf.equals(dstBuf);
-  } catch {
-    return false;
-  }
-}
-
-function tryUnlink(filePath) {
-  try { unlinkSync(filePath); } catch { /* non-fatal */ }
-}
-
-function migrateHnswIndex(projectRoot) {
-  const src = legacyHnswIndexPath(projectRoot);
-  const dst = hnswIndexPath(projectRoot);
-
-  if (!existsSync(src)) return false;
-  if (existsSync(dst)) return false;
-
-  try {
-    mkdirSync(dirname(dst), { recursive: true });
-    copyFileSync(src, dst);
-  } catch {
-    tryUnlink(dst);
-    return false;
-  }
-
-  if (!verifyByteEqual(src, dst)) {
-    tryUnlink(dst);
-    return false;
-  }
-
-  tryUnlink(src);
-  return true;
-}
-
-/**
- * One-time relocation of memory DB from `.swarm/memory.db` → `.moflo/moflo.db`
- * (story #727). Idempotent. See moflo-paths.ts for the full contract and the
- * SQLite-header / byte-equal verification rationale.
- *
- * MUST run before any long-lived sql.js consumer (MCP server, daemon) opens
- * the DB — sql.js dumps the whole snapshot on every flush and would clobber
- * the relocated file. Launcher section 0b is the safe boundary.
- */
-export function migrateMemoryDbToMoflo(projectRoot) {
-  const target = memoryDbPath(projectRoot);
-  if (existsSync(target)) return { migrated: false, reason: 'target-exists' };
-
-  const source = legacyMemoryDbPath(projectRoot);
-  if (!existsSync(source)) return { migrated: false, reason: 'no-legacy' };
-
-  try {
-    mkdirSync(dirname(target), { recursive: true });
-  } catch {
-    return { migrated: false, reason: 'copy-failed' };
-  }
-
-  try {
-    copyFileSync(source, target);
-  } catch {
-    tryUnlink(target);
-    return { migrated: false, reason: 'copy-failed' };
-  }
-
-  if (!verifyByteEqual(source, target) || !looksLikeSqliteFile(target)) {
-    tryUnlink(target);
-    return { migrated: false, reason: 'verify-failed' };
-  }
-
-  const hnswMoved = migrateHnswIndex(projectRoot);
-
-  try {
-    renameSync(source, legacyMemoryDbBakPath(projectRoot));
-  } catch {
-    return { migrated: true, reason: 'rename-failed', hnswMoved };
-  }
-
-  return { migrated: true, hnswMoved };
-}

package/bin/lib/process-manager.mjs

@@ -217,12 +217,17 @@ export function createProcessManager(root) {
         if (!isAlive(entry.pid)) continue;
         try {
           if (process.platform === 'win32') {
-            execFileSync('taskkill', ['/T', '/F', '/PID', String(entry.pid)], { windowsHide: true, timeout: 5000 });
+            execFileSync('taskkill', ['/T', '/F', '/PID', String(entry.pid)], { windowsHide: true, timeout: 10000 });
           } else {
             process.kill(entry.pid, 'SIGTERM');
           }
           killed++;
-        } catch { /* already gone */ }
+        } catch {
+          // Wrapper call threw (e.g. taskkill exceeded its timeout under load).
+          // The process itself may still have been terminated — count it as
+          // killed if it is no longer alive.
+          if (!isAlive(entry.pid)) killed++;
+        }
       }
 
       // Clear registry and lock