@massu/core 1.2.1 → 1.4.0-soak.0

package/src/watch/paths.ts

@@ -0,0 +1,279 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+/**
+ * Watch glob derivation.
+ *
+ * Watch surface = manifest files (always) + source directories
+ * (from massu.config.yaml's paths.* and framework.languages.*.source_dirs,
+ * or fallback safe-default globs when both absent), bounded by exclusion
+ * globs.
+ */
+
+import fastGlob from 'fast-glob';
+import type { Config } from '../config.ts';
+
+export const ALWAYS_WATCH_FILES = [
+  'package.json',
+  'pyproject.toml',
+  'Cargo.toml',
+  'go.mod',
+  'Gemfile',
+  '*.csproj',
+  'mix.exs',
+  'requirements*.txt',
+  'setup.py',
+] as const;
+
+export const FALLBACK_SOURCE_GLOBS = [
+  'src/**',
+  'app/**',
+  'apps/**',
+  'packages/**',
+  'lib/**',
+  'cmd/**',
+] as const;
+
+export const DEFAULT_EXCLUSIONS = [
+  '**/node_modules/**',
+  '**/.venv/**',
+  '**/venv/**',
+  '**/target/**',
+  '**/build/**',
+  '**/dist/**',
+  '**/.git/**',
+  '**/.massu/**',
+  '**/.claude/**',
+  '**/__pycache__/**',
+  '**/.pytest_cache/**',
+  '**/.mypy_cache/**',
+  // Plan 3a hotfix 2026-05-02: high-churn directories that are never
+  // legitimate stack-detection inputs and produced sustained 30-100% CPU
+  // when watched on Hedge (62K files / 42 GB tree).
+  '**/.next/**',
+  '**/coverage/**',
+  '**/logs/**',
+  '**/*.log',
+  // Runtime data dirs. Convention across Python/JS/Rust ecosystems is
+  // that `data/` holds runtime artifacts (caches, snapshots, model
+  // checkpoints, downloaded fixtures) that change frequently but are
+  // never stack-detection inputs. Hedge had 135K files in
+  // apps/ai-service/data alone, dwarfing legitimate source. If a
+  // project genuinely uses `data/` for source content, opt into
+  // `watch.scope: 'full'` and `watch.paths_full_root_opt_in: true`.
+  '**/data/**',
+  // Iter-7 fix: editor temp files inside watched dirs fire spurious chokidar
+  // events and inflate the storm-detection counter without representing real
+  // stack changes. Cover the most common cases:
+  //   *.swp / *.swo / 4913   -> vim atomic-write probe + swap files
+  //   .#*                    -> emacs lockfiles
+  //   *~                     -> gedit / many editors backup
+  //   .DS_Store              -> macOS Finder metadata
+  '**/*.swp',
+  '**/*.swo',
+  '**/4913',
+  '**/.#*',
+  '**/*~',
+  '**/.DS_Store',
+];
+
+/**
+ * Plan 3a hotfix 2026-05-02: explicit root-watch sentinels. Any of these
+ * appearing in `framework.languages.*.source_dirs` is treated as the user
+ * asking to watch the entire repo root, which requires
+ * `watch.paths_full_root_opt_in: true` to override the file-count cap.
+ *
+ * Why these specifically: the toGlob() helper turns `'.'` into `'./**'`
+ * and leaves `'**'` / `'./**'` / `'./'` as-is. Each effectively makes
+ * chokidar walk the toplevel — silently, without warning — defeating
+ * the scope='paths' bound. We treat them as semantically equivalent to
+ * `scope: 'full'`.
+ */
+export const ROOT_WATCH_SENTINELS = ['.', './', '**', './**', '*'] as const;
+
+export function isRootWatchSentinel(dir: string): boolean {
+  return (ROOT_WATCH_SENTINELS as readonly string[]).includes(dir);
+}
+
+export class WatchSurfaceTooLargeError extends Error {
+  constructor(public readonly fileCount: number, public readonly cap: number) {
+    super(
+      `massu watch refuses to start: would monitor ${fileCount} files ` +
+      `(cap is ${cap}). Narrow framework.languages.*.source_dirs in ` +
+      `massu.config.yaml, or set watch.paths_full_root_opt_in: true (and ` +
+      `watch.max_watched_files: ${fileCount + 1000}) if root-level watching ` +
+      `is genuinely required. Note: monitoring more than 10K files routinely ` +
+      `produces 30-100% steady CPU under normal repo activity.`
+    );
+    this.name = 'WatchSurfaceTooLargeError';
+  }
+}
+
+export interface DerivedWatchGlobs {
+  /** Globs/files to watch. */
+  watch: string[];
+  /** Globs to exclude. */
+  ignore: string[];
+  /** True when fallback globs were used (because config didn't declare any source paths). */
+  usedFallback: boolean;
+  /** Effective scope after considering root sentinels in source_dirs. */
+  effectiveScope: 'paths' | 'full';
+  /** True when a root sentinel ('.', '**', etc.) was detected in source_dirs and promoted to full scope. */
+  rootWatchDetected: boolean;
+}
+
+/**
+ * Build the watch + ignore glob set for chokidar from a loaded Config.
+ * Returns project-relative globs; the daemon resolves them against its root.
+ *
+ * `watch.scope` (Plan 3a §167 + §251 risk #1):
+ *   - `'paths'` (default) — watch only declared `paths.*` + `framework.languages.*.source_dirs`
+ *     (or fallback safe-default globs when none declared). Bounded watch surface.
+ *   - `'full'` — watch the entire project root (`'**'`) bounded by exclusion globs.
+ *     Opt-in for users on small repos who want every file under the toplevel
+ *     to count. NOT recommended for huge (>10K-file) repos.
+ */
+export function deriveWatchGlobs(config: Config): DerivedWatchGlobs {
+  const sourceDirs = new Set<string>();
+  const explicitScope = config.watch?.scope ?? 'paths';
+  let rootWatchDetected = false;
+
+  if (explicitScope === 'full') {
+    // Full-repo opt-in. Just watch '**' and rely on DEFAULT_EXCLUSIONS for
+    // node_modules / .git / .massu / .claude / build dirs.
+    sourceDirs.add('**');
+    return {
+      watch: [...ALWAYS_WATCH_FILES, ...sourceDirs],
+      ignore: [...DEFAULT_EXCLUSIONS],
+      usedFallback: false,
+      effectiveScope: 'full',
+      rootWatchDetected: false,
+    };
+  }
+
+  if (config.paths.source && typeof config.paths.source === 'string') {
+    if (isRootWatchSentinel(config.paths.source)) {
+      rootWatchDetected = true;
+    } else {
+      sourceDirs.add(toGlob(config.paths.source));
+    }
+  }
+
+  const langs = config.framework.languages;
+  if (langs && typeof langs === 'object') {
+    for (const langEntry of Object.values(langs)) {
+      // Defensive: some entries may not include source_dirs. Use property
+      // narrowing instead of an `as unknown as Shape` cast.
+      if (
+        langEntry &&
+        typeof langEntry === 'object' &&
+        'source_dirs' in langEntry
+      ) {
+        const dirs = (langEntry as { source_dirs?: unknown }).source_dirs;
+        if (Array.isArray(dirs)) {
+          for (const d of dirs) {
+            if (typeof d !== 'string' || !d) continue;
+            if (isRootWatchSentinel(d)) {
+              rootWatchDetected = true;
+            } else {
+              sourceDirs.add(toGlob(d));
+            }
+          }
+        }
+      }
+    }
+  }
+
+  const usedFallback = sourceDirs.size === 0 && !rootWatchDetected;
+  if (usedFallback) {
+    for (const g of FALLBACK_SOURCE_GLOBS) sourceDirs.add(g);
+  }
+
+  // Plan 3a hotfix: a root sentinel in source_dirs is semantically the same
+  // as scope='full'. Promote it. The daemon enforces the file-count cap on
+  // top of this; an unintentional `.` in source_dirs gets caught there.
+  let effectiveScope: 'paths' | 'full' = explicitScope;
+  if (rootWatchDetected) {
+    sourceDirs.add('**');
+    effectiveScope = 'full';
+  }
+
+  return {
+    watch: [...ALWAYS_WATCH_FILES, ...sourceDirs],
+    ignore: [...DEFAULT_EXCLUSIONS],
+    usedFallback,
+    effectiveScope,
+    rootWatchDetected,
+  };
+}
+
+function toGlob(dir: string): string {
+  if (dir.endsWith('/**') || dir.includes('*')) return dir;
+  return dir.replace(/\/+$/, '') + '/**';
+}
+
+/**
+ * Plan 3a hotfix 2026-05-02: count files matching the derived watch globs
+ * (after exclusions) without walking past the cap. Used as a startup
+ * preflight in the daemon — refuses to start if count > cap and the user
+ * hasn't set `watch.paths_full_root_opt_in: true`.
+ *
+ * Why upfront vs after chokidar: chokidar.getWatched() requires waiting
+ * for the `ready` event, which on a 62K-file tree takes 30+ seconds AND
+ * does the full walk we're trying to avoid. fast-glob with `onlyFiles`
+ * + early-exit via the iterator pattern walks once, bails when we've
+ * counted enough to know we're over the cap.
+ *
+ * Returns Infinity if more than `cap + 1` files exist (signals "exceeds")
+ * to avoid walking the full tree just to produce an exact count.
+ */
+export async function countWatchSurface(
+  watch: readonly string[],
+  ignore: readonly string[],
+  cwd: string,
+  cap: number
+): Promise<number> {
+  // fast-glob's stream API yields one path per event, allowing early-exit.
+  const stream = fastGlob.stream(watch as string[], {
+    cwd,
+    ignore: ignore as string[],
+    onlyFiles: true,
+    dot: false,
+    followSymbolicLinks: false,
+    suppressErrors: true,
+  });
+  let count = 0;
+  for await (const _ of stream) {
+    count += 1;
+    if (count > cap) {
+      // We've proven the surface exceeds the cap. Cancel the iterator and
+      // return a sentinel value. The caller only needs the exceeds-or-not
+      // signal (plus the cap) to produce a useful error.
+      // @ts-expect-error fast-glob stream is a NodeJS.ReadableStream
+      stream.destroy?.();
+      return Infinity;
+    }
+  }
+  return count;
+}
+
+/**
+ * Plan 3a hotfix 2026-05-02: enforce the watch.max_watched_files cap.
+ * Throws WatchSurfaceTooLargeError if the count exceeds the cap and the
+ * user has not opted in via `watch.paths_full_root_opt_in: true`.
+ *
+ * Returns the actual count (or Infinity if early-exit tripped) so the
+ * daemon can log the surface size at startup for observability.
+ */
+export async function enforceWatchSurfaceCap(
+  globs: DerivedWatchGlobs,
+  cwd: string,
+  cap: number,
+  optedIn: boolean
+): Promise<number> {
+  const count = await countWatchSurface(globs.watch, globs.ignore, cwd, cap);
+  if (count > cap && !optedIn) {
+    throw new WatchSurfaceTooLargeError(count, cap);
+  }
+  return count;
+}

package/src/watch/state.ts

@@ -0,0 +1,178 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+/**
+ * Watcher state persistence — `.massu/watch-state.json`.
+ *
+ * Crash-recovery contract: every mutation goes through writeStateAtomic()
+ * (write `.tmp` → fsyncSync → renameSync). At any kill -9 point either the
+ * old state or the new state survives intact (POSIX rename atomicity).
+ *
+ * Schema versioning: top-level `schema_version` field. Migrators in
+ * STATE_MIGRATORS bump from `from` → `from + 1` until MAX_SUPPORTED.
+ */
+
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, writeSync } from 'fs';
+import { dirname, resolve } from 'path';
+
+export const MIN_SUPPORTED_SCHEMA_VERSION = 1;
+export const MAX_SUPPORTED_SCHEMA_VERSION = 1;
+
+export interface WatchState {
+  schema_version: number;
+  lastFingerprint: string | null;
+  lastRefreshAt: string | null;
+  lastError: string | null;
+  daemonPid: number | null;
+  startedAt: string | null;
+  tickedAt: string | null;
+}
+
+export const DEFAULT_STATE: WatchState = {
+  schema_version: MAX_SUPPORTED_SCHEMA_VERSION,
+  lastFingerprint: null,
+  lastRefreshAt: null,
+  lastError: null,
+  daemonPid: null,
+  startedAt: null,
+  tickedAt: null,
+};
+
+/**
+ * Bumps state from a known prior version to `from + 1`. Empty at ship.
+ * When a future schema bump lands, register a migrator here for every
+ * `from` in [MIN_SUPPORTED_SCHEMA_VERSION..MAX_SUPPORTED_SCHEMA_VERSION-1].
+ */
+export const STATE_MIGRATORS: Record<number, (old: Record<string, unknown>) => Record<string, unknown>> = {};
+
+export function watchStatePath(projectRoot: string): string {
+  return resolve(projectRoot, '.massu', 'watch-state.json');
+}
+
+export function backupStatePath(projectRoot: string): string {
+  return resolve(projectRoot, '.massu', 'watch-state.v0.bak.json');
+}
+
+export class WatchStateNewerError extends Error {
+  constructor(public stateVersion: number, public daemonMax: number) {
+    super(`watch-state from newer massu version (v=${stateVersion}, daemon supports v=${daemonMax}); refusing to overwrite — upgrade massu or delete .massu/watch-state.json`);
+    this.name = 'WatchStateNewerError';
+  }
+}
+
+/**
+ * Read state from disk. Behavior on schema mismatch:
+ *  - missing schema_version: archive to backup path, return DEFAULT_STATE
+ *  - older than supported: run STATE_MIGRATORS in sequence
+ *  - newer than supported: throw WatchStateNewerError
+ */
+export function readState(projectRoot: string): WatchState {
+  const path = watchStatePath(projectRoot);
+  if (!existsSync(path)) return { ...DEFAULT_STATE };
+
+  // Read once; reuse the content for archive-on-corrupt to avoid a redundant
+  // disk read in the failure path (iter-9 simplify finding E1).
+  const content = readFileSync(path, 'utf-8');
+
+  let raw: unknown;
+  try {
+    raw = JSON.parse(content);
+  } catch {
+    archiveCorrupt(projectRoot, content);
+    return { ...DEFAULT_STATE };
+  }
+
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+    return { ...DEFAULT_STATE };
+  }
+
+  const obj = raw as Record<string, unknown>;
+  const ver = obj.schema_version;
+
+  if (typeof ver !== 'number') {
+    archiveCorrupt(projectRoot, content);
+    return { ...DEFAULT_STATE };
+  }
+
+  if (ver > MAX_SUPPORTED_SCHEMA_VERSION) {
+    throw new WatchStateNewerError(ver, MAX_SUPPORTED_SCHEMA_VERSION);
+  }
+
+  let migrated: Record<string, unknown> = obj;
+  for (let v = ver; v < MAX_SUPPORTED_SCHEMA_VERSION; v++) {
+    const migrator = STATE_MIGRATORS[v];
+    if (!migrator) {
+      throw new Error(
+        `watch-state.json: missing migrator for schema_version ${v} -> ${v + 1}. ` +
+        `Daemon supports up to v=${MAX_SUPPORTED_SCHEMA_VERSION}; this is a massu bug. ` +
+        `Workaround: delete .massu/watch-state.json (the daemon will rebuild it on next start).`,
+      );
+    }
+    migrated = migrator(migrated);
+  }
+
+  return {
+    schema_version: MAX_SUPPORTED_SCHEMA_VERSION,
+    lastFingerprint: typeof migrated.lastFingerprint === 'string' ? migrated.lastFingerprint : null,
+    lastRefreshAt: typeof migrated.lastRefreshAt === 'string' ? migrated.lastRefreshAt : null,
+    lastError: typeof migrated.lastError === 'string' ? migrated.lastError : null,
+    daemonPid: typeof migrated.daemonPid === 'number' ? migrated.daemonPid : null,
+    startedAt: typeof migrated.startedAt === 'string' ? migrated.startedAt : null,
+    tickedAt: typeof migrated.tickedAt === 'string' ? migrated.tickedAt : null,
+  };
+}
+
+function archiveCorrupt(projectRoot: string, content: string): void {
+  const bak = backupStatePath(projectRoot);
+  mkdirSync(dirname(bak), { recursive: true });
+  writeFileSync(bak, content, 'utf-8');
+}
+
+// Per-process counter for unique temp filenames. Combined with PID this
+// makes the temp path unique across concurrent writers in the same project.
+let writeStateAtomicCounter = 0;
+
+/**
+ * Atomic write: tmp + fsync + rename. Survives kill -9 at any point.
+ *
+ * The temp filename is `<path>.<pid>.<counter>.tmp` so that two concurrent
+ * processes (e.g. foreground daemon + a `massu watch --apply-now` racing
+ * against it) never share the same temp file. POSIX rename is atomic per
+ * (target) so the final state file always reflects exactly one of the
+ * concurrent writers — never a torn write.
+ */
+export function writeStateAtomic(projectRoot: string, state: WatchState): void {
+  const path = watchStatePath(projectRoot);
+  writeStateAtomicCounter = (writeStateAtomicCounter + 1) >>> 0;
+  const tmp = `${path}.${process.pid}.${writeStateAtomicCounter}.tmp`;
+  mkdirSync(dirname(path), { recursive: true });
+
+  // Iter-8 fix: clean up tmp on error. Without this, a writeSync/fsyncSync
+  // failure (e.g., ENOSPC, EIO mid-write) leaves a `.<pid>.<counter>.tmp`
+  // straggler in `.massu/` for every failed attempt. Mirrors the cleanup-on-
+  // error pattern used by `atomicWriteFile` in install-commands.ts.
+  let renamed = false;
+  try {
+    const fd = openSync(tmp, 'w');
+    try {
+      const buf = Buffer.from(JSON.stringify(state, null, 2) + '\n', 'utf-8');
+      writeSync(fd, buf, 0, buf.length, 0);
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+    renameSync(tmp, path);
+    renamed = true;
+  } finally {
+    if (!renamed && existsSync(tmp)) {
+      try { rmSync(tmp, { force: true }); } catch { /* best-effort */ }
+    }
+  }
+}
+
+export function updateState(projectRoot: string, patch: Partial<WatchState>): WatchState {
+  const current = readState(projectRoot);
+  const next: WatchState = { ...current, ...patch, schema_version: MAX_SUPPORTED_SCHEMA_VERSION };
+  writeStateAtomic(projectRoot, next);
+  return next;
+}