moflo 4.10.20 → 4.10.21

package/bin/index-all.mjs

@@ -78,7 +78,7 @@ function isIndexEnabled(key) {
     if (existsSync(yamlPath)) {
       try {
         const content = readFileSync(yamlPath, 'utf-8');
-        for (const k of ['guidance', 'code_map', 'tests', 'patterns']) {
+        for (const k of ['guidance', 'code_map', 'tests', 'patterns', 'reference']) {
           const re = new RegExp(`auto_index:\\s*\\n(?:.*\\n)*?\\s+${k}:\\s*(true|false)`);
           const match = content.match(re);
           _autoIndexFlags[k] = match ? match[1] !== 'false' : true;
@@ -182,6 +182,7 @@ function buildStepPlan() {
   consider('code-map',       'code_map', 'generate-code-map.mjs', 'flo-codemap', ['--no-embeddings'], 180_000);
   consider('test-index',     'tests',    'index-tests.mjs',    'flo-testmap', ['--no-embeddings']);
   consider('patterns-index', 'patterns', 'index-patterns.mjs', 'flo-patterns', []);
+  consider('reference-index', 'reference', 'index-reference.mjs', 'flo-reference', []);
 
   // Pretrain extracts patterns from the repo via the CLI subcommand. No
   // direct script — invoke through the local flo binary.

package/bin/index-reference.mjs

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+/**
+ * Index installed library docs into moflo memory under the `reference` namespace.
+ *
+ * Native, version-pinned alternative to a hosted docs MCP (e.g. Context7). For
+ * every package the repo DIRECTLY depends on, this reads the docs that already
+ * sit in `node_modules` — the entry `.d.ts` type surface and the README — chunks
+ * them, and stores them keyed on the INSTALLED version. Retrieval is free: the
+ * chunks land in the same HNSW store every other namespace uses, so the agent's
+ * mandated `memory_search` first action surfaces them with navigation crumbs.
+ *
+ * Why this shape (see issue #1184):
+ *   - Version-correct by construction — the resolved folder IS the version; we
+ *     read `node_modules/<pkg>/package.json.version`, so it works identically
+ *     across npm/yarn/pnpm/bun with no lockfile parsing.
+ *   - Zero network, fully offline — `fs` reads only.
+ *   - Cross-platform — `path.join` only, no shelling out.
+ *   - Bounded — DIRECT deps only (not the transitive tree), with per-doc size
+ *     and chunk caps so one mega-package can't dominate the index.
+ *   - Graceful — a package with no README/types contributes nothing; never an
+ *     error. Wrong docs are worse than none.
+ *
+ * The pure discovery/chunking/entry-shaping logic lives in
+ * `./lib/reference-docs.mjs` (unit-tested); this file is the orchestrator that
+ * owns the DB write, the incremental-diff gate, and the background embed spawn.
+ *
+ * Usage:
+ *   node node_modules/moflo/bin/index-reference.mjs             # Incremental
+ *   node node_modules/moflo/bin/index-reference.mjs --force     # Full reindex
+ *   node node_modules/moflo/bin/index-reference.mjs --verbose   # Detailed logging
+ *   node node_modules/moflo/bin/index-reference.mjs --stats     # Print stats and exit
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { createHash } from 'crypto';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { resolveMofloBin } from './lib/resolve-bin.mjs';
+import { memoryDbPath, MOFLO_DIR, findProjectRoot } from './lib/moflo-paths.mjs';
+import { openBackend } from './lib/get-backend.mjs';
+import { applyIncrementalChunks, computeContentListHash } from './lib/incremental-write.mjs';
+import { createProcessManager } from './lib/process-manager.mjs';
+import { collectReferenceDocs, buildDocEntries } from './lib/reference-docs.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const projectRoot = findProjectRoot();
+const NAMESPACE = 'reference';
+const DB_PATH = memoryDbPath(projectRoot);
+const HASH_CACHE_PATH = resolve(projectRoot, MOFLO_DIR, 'reference-hash.txt');
+
+const args = process.argv.slice(2);
+const force = args.includes('--force');
+const verbose = args.includes('--verbose') || args.includes('-v');
+const statsOnly = args.includes('--stats');
+
+function log(msg) { console.log(`[index-reference] ${msg}`); }
+function debug(msg) { if (verbose) console.log(`[index-reference]   ${msg}`); }
+
+// ---------------------------------------------------------------------------
+// Database helpers — identical shape to the other indexers (#745 incremental)
+// ---------------------------------------------------------------------------
+
+function ensureDbDir() {
+  const dir = dirname(DB_PATH);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+}
+
+async function getDb() {
+  ensureDbDir();
+  const db = await openBackend(projectRoot, { create: true });
+  db.run(`
+    CREATE TABLE IF NOT EXISTS memory_entries (
+      id TEXT PRIMARY KEY,
+      key TEXT NOT NULL,
+      namespace TEXT DEFAULT 'default',
+      content TEXT NOT NULL,
+      type TEXT DEFAULT 'semantic',
+      embedding TEXT,
+      embedding_model TEXT DEFAULT 'local',
+      embedding_dimensions INTEGER,
+      tags TEXT,
+      metadata TEXT,
+      owner_id TEXT,
+      created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+      updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+      expires_at INTEGER,
+      last_accessed_at INTEGER,
+      access_count INTEGER DEFAULT 0,
+      status TEXT DEFAULT 'active',
+      UNIQUE(namespace, key)
+    )
+  `);
+  db.run(`CREATE INDEX IF NOT EXISTS idx_memory_key_ns ON memory_entries(key, namespace)`);
+  db.run(`CREATE INDEX IF NOT EXISTS idx_memory_namespace ON memory_entries(namespace)`);
+  return db;
+}
+
+function countNamespace(db) {
+  const stmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE namespace = ?`);
+  stmt.bind([NAMESPACE]);
+  let count = 0;
+  if (stmt.step()) count = stmt.getAsObject().cnt;
+  stmt.free();
+  return count;
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const startTime = Date.now();
+
+  const { packages, docFiles, depCount } = collectReferenceDocs(projectRoot);
+
+  if (depCount === 0) {
+    log('No dependencies found in package.json (nothing to ground)');
+    return;
+  }
+
+  if (statsOnly) {
+    const db = await getDb();
+    const count = countNamespace(db);
+    db.close();
+    log(`${packages.length} packages with docs (of ${depCount} deps), ${count} chunks in reference namespace`);
+    return;
+  }
+
+  if (packages.length === 0) {
+    log(`No installed docs found across ${depCount} dependencies`);
+    return;
+  }
+
+  // Outer gate — content hash over the doc files combined with each resolved
+  // name@version (so a version bump with byte-identical docs still re-keys).
+  // The version line is folded straight into the digest — no sidecar file — so
+  // the gate is deterministic and side-effect-free. Skips the whole
+  // extract+write pipeline when nothing changed (#746).
+  const versionLine = packages.map((p) => `${p.name}@${p.version}`).join(',');
+  const currentHash = createHash('sha256')
+    .update(versionLine)
+    .update('\n')
+    .update(computeContentListHash(docFiles))
+    .digest('hex');
+
+  if (!force && existsSync(HASH_CACHE_PATH)) {
+    const cached = readFileSync(HASH_CACHE_PATH, 'utf-8').trim();
+    if (cached === currentHash) {
+      log('No dependency-doc changes detected (use --force to reindex)');
+      return;
+    }
+  }
+
+  // Extract chunks from every resolved package.
+  const allEntries = [];
+  let packagesIndexed = 0;
+  for (const pkg of packages) {
+    let pkgEntries = 0;
+    if (pkg.readmePath) {
+      try {
+        const entries = buildDocEntries(pkg, 'readme', readFileSync(pkg.readmePath, 'utf-8'));
+        allEntries.push(...entries);
+        pkgEntries += entries.length;
+      } catch { /* unreadable README — skip */ }
+    }
+    if (pkg.typesPath) {
+      try {
+        const entries = buildDocEntries(pkg, 'types', readFileSync(pkg.typesPath, 'utf-8'));
+        allEntries.push(...entries);
+        pkgEntries += entries.length;
+      } catch { /* unreadable .d.ts — skip */ }
+    }
+    if (pkgEntries > 0) packagesIndexed++;
+    debug(`${pkg.name}@${pkg.version}: ${pkgEntries} chunks`);
+  }
+
+  log(`Extracted ${allEntries.length} doc chunks from ${packagesIndexed} packages`);
+
+  // Content-aware diff — unchanged rows keep their embeddings; orphaned chunks
+  // (including every chunk of an upgraded package's old version) are swept.
+  const db = await getDb();
+  const counts = applyIncrementalChunks(db, NAMESPACE, allEntries);
+  if (counts.inserted + counts.updated + counts.removed > 0) db.save();
+  db.close();
+
+  log(
+    `Diff: ${counts.inserted} new, ${counts.updated} updated, ` +
+    `${counts.unchanged} unchanged, ${counts.removed} removed`,
+  );
+
+  writeFileSync(HASH_CACHE_PATH, currentHash, 'utf-8');
+
+  // Embed the new/changed rows in the background, registered with the shared
+  // ProcessManager so doctor's zombie scan allowlists it and teardown reaps it.
+  // The namespace-derived label dedupes a second index-reference spawn within
+  // the lock window; build-embeddings only fills rows whose embedding IS NULL,
+  // so index-all's later global pass won't re-embed these.
+  try {
+    const embeddingScript = resolveMofloBin(
+      projectRoot, 'flo-embeddings', 'build-embeddings.mjs', { includeDevFallback: true },
+    );
+    if (embeddingScript) {
+      const pm = createProcessManager(projectRoot);
+      const result = pm.spawn('node', [embeddingScript, '--namespace', NAMESPACE], `build-embeddings-${NAMESPACE}`);
+      if (result.skipped) {
+        debug(`Embedding generation already running (PID: ${result.pid})`);
+      } else if (result.pid) {
+        debug(`Embedding generation started in background (PID: ${result.pid})`);
+      }
+    }
+  } catch (err) { debug(`embedding spawn skipped: ${err.message}`); }
+
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  log(`Done in ${elapsed}s — ${allEntries.length} reference chunks written`);
+}
+
+main().catch(err => {
+  log(`Error: ${err.message}`);
+  process.exit(1);
+});

package/bin/lib/file-sync.mjs

@@ -28,13 +28,14 @@ import {
   copyFileSync,
   existsSync,
   mkdirSync,
+  readdirSync,
   readFileSync,
   renameSync,
   statSync,
   unlinkSync,
 } from 'node:fs';
 import { createHash } from 'node:crypto';
-import { dirname } from 'node:path';
+import { dirname, relative, resolve } from 'node:path';
 
 export const TRANSIENT_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
 export const RETRY_BACKOFF_MS = [50, 200, 800];
@@ -198,3 +199,51 @@ export function makeSyncer({ onSuccess } = {}) {
     isCircuitOpen: () => circuitOpen,
   };
 }
+
+/**
+ * Recursively sync every `.md` under `srcDir` into `<projectRoot>/<destPrefix>`,
+ * skipping any entry whose top-level directory is listed in `excludeTopLevel`.
+ *
+ * The session-start launcher uses this to mirror `node_modules/moflo/.claude/
+ * agents` and `.claude/skills` into the consumer project. `excludeTopLevel`
+ * keeps moflo-internal skills (`bin/lib/internal-skills.mjs`) out of consumer
+ * installs — without it the blanket copy would land `/publish` and `/reset-epic`
+ * in every consumer (they ship in the tarball but must not be installed).
+ *
+ * Each file is copied through `syncFile` so the manifest, hash-skip, and
+ * retry/breaker behaviour are identical to every other synced path.
+ *
+ * @param {string} srcDir   Absolute source directory (e.g. node_modules/moflo/.claude/skills).
+ * @param {string} destPrefix  Project-relative destination prefix (e.g. '.claude/skills').
+ * @param {object} opts
+ * @param {string} opts.projectRoot  Consumer project root the destPrefix is resolved against.
+ * @param {(src: string, dest: string, key: string) => Promise<unknown>} opts.syncFile  From makeSyncer().
+ * @param {Set<string>|string[]} [opts.excludeTopLevel]  Top-level dir names to skip.
+ * @param {(message: string) => void} [opts.onWarn]  Non-fatal warning sink.
+ */
+export async function syncDirRecursive(srcDir, destPrefix, { projectRoot, syncFile, excludeTopLevel, onWarn } = {}) {
+  if (!existsSync(srcDir)) return;
+  let entries;
+  try {
+    entries = readdirSync(srcDir, { recursive: true, withFileTypes: true });
+  } catch (err) {
+    onWarn?.(`${destPrefix} readdir failed (${errMessage(err)})`);
+    return;
+  }
+  const exclude = excludeTopLevel instanceof Set
+    ? excludeTopLevel
+    : new Set(excludeTopLevel || []);
+  for (const entry of entries) {
+    if (!entry.isFile()) continue;
+    if (!entry.name.toLowerCase().endsWith('.md')) continue;
+    const parent = entry.parentPath || entry.path || srcDir;
+    const absSrc = resolve(parent, entry.name);
+    // path.relative (not slice) so a trailing separator on srcDir can't shift
+    // the top-level segment and silently defeat the exclusion check.
+    const rel = relative(srcDir, absSrc).split(/[\\/]/).join('/');
+    if (exclude.size > 0 && exclude.has(rel.split('/')[0])) continue;
+    const absDest = resolve(projectRoot, destPrefix, rel);
+    // syncFile owns dir creation (mkdir recursive) — no outer mkdir needed.
+    await syncFile(absSrc, absDest, `${destPrefix}/${rel}`);
+  }
+}

package/bin/lib/hook-io.mjs

@@ -0,0 +1,63 @@
+/**
+ * Shared hook I/O primitives for moflo's bin/*.mjs hook handlers.
+ *
+ * Extracted (#1198) so the UserPromptSubmit/Stop capture hook
+ * (meditate-capture.mjs) and the passive session-continuity Stop hook
+ * (session-continuity.mjs) share ONE implementation of "read the hook's JSON
+ * stdin" — a bug in the bounded-read logic gets fixed once, not per copy.
+ *
+ * Cross-platform (Rule #1): Node fs primitives only; no shell.
+ */
+
+import { existsSync, openSync, readSync, closeSync, statSync, readFileSync } from 'fs';
+
+/**
+ * Read a hook's JSON stdin (session_id, transcript_path, prompt, …). Bounded by
+ * a 500ms cap so a missing/withheld stdin can never hang the hook; the timer is
+ * cleared on a normal end so the process exits immediately instead of lingering
+ * up to 500ms. Never throws — returns {} on any parse/IO error.
+ *
+ * @returns {Promise<Record<string, any>>}
+ */
+export async function readHookStdin() {
+  if (process.stdin.isTTY) return {};
+  return new Promise((res) => {
+    let data = '';
+    let done = false;
+    let timer = null;
+    const parse = (s) => { try { return s ? JSON.parse(s) : {}; } catch { return {}; } };
+    const finish = () => { if (done) return; done = true; if (timer) clearTimeout(timer); res(parse(data)); };
+    process.stdin.setEncoding('utf-8');
+    process.stdin.on('data', (c) => { if (!done) data += c; });
+    process.stdin.on('end', finish);
+    process.stdin.on('error', finish);
+    timer = setTimeout(finish, 500);
+  });
+}
+
+/**
+ * Read the last `bytes` of a file as UTF-8. Returns '' on any error. The first
+ * (likely partial) line is the caller's concern — JSONL/transcript parsers
+ * tolerate a truncated leading line.
+ *
+ * @param {string} path
+ * @param {number} bytes
+ * @returns {string}
+ */
+export function readFileTail(path, bytes) {
+  try {
+    if (!path || !existsSync(path)) return '';
+    const size = statSync(path).size;
+    if (size <= bytes) return readFileSync(path, 'utf-8');
+    const fd = openSync(path, 'r');
+    try {
+      const buf = Buffer.alloc(bytes);
+      readSync(fd, buf, 0, bytes, size - bytes);
+      return buf.toString('utf-8');
+    } finally {
+      closeSync(fd);
+    }
+  } catch {
+    return '';
+  }
+}

package/bin/lib/internal-skills.mjs

@@ -0,0 +1,16 @@
+/**
+ * Skills that ship in the npm tarball (under `node_modules/moflo/.claude/skills/`)
+ * but must NEVER be installed into consumer projects — strictly moflo-internal
+ * dev tooling. `/publish` bumps moflo's own version and publishes to npm;
+ * `/reset-epic` torches epic test data. Both are meaningless or harmful in a
+ * consumer repo.
+ *
+ * The session-start launcher's recursive skills sync (`syncDirRecursive` in
+ * `file-sync.mjs`) copies every shipped skill into the consumer on each run, so
+ * it MUST skip these. The canonical TypeScript copy is `INTERNAL_SKILLS` in
+ * `src/cli/init/executor.ts` (used by `flo init`). The launcher is a plain
+ * `.mjs` and can't import that TS const across the dist/source depth boundary,
+ * so this leaf mirrors it. `tests/bin/internal-skills-parity.test.ts` asserts
+ * the two lists never drift.
+ */
+export const INTERNAL_SKILLS = ['publish', 'reset-epic'];