@indigoai-us/hq-cloud 6.14.44 → 6.14.45

package/src/watcher.ts

@@ -12,6 +12,11 @@ import { readFile, stat } from "node:fs/promises";
 import * as path from "path";
 import { watch } from "chokidar";
 import { createIgnoreFilter } from "./ignore.js";
+import {
+  isCoveredByRecursiveRoot,
+  planWatchRoots,
+  type WatchRootPlan,
+} from "./watch-roots.js";
 import { vaultKeyForLocalPath } from "./local-path-codec.js";
 import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
 import {
@@ -340,6 +345,12 @@ export interface WatchBackend {
   close(): void;
   /** Native recursive watches need rename-kind state; chokidar does not. */
   needsKnownKinds: boolean;
+  /**
+   * True when the backend already populated those known kinds through
+   * `onDirectorySeen` while planning its watch roots, so {@link TreeWatcher}
+   * must NOT walk the tree a second time to seed them.
+   */
+  seededKnownKinds?: boolean;
   /** Directories/OS handles admitted by this backend so degradation is auditable. */
   watchedPathCount(): number;
 }
@@ -361,6 +372,11 @@ export interface TreeWatchBackendOptions {
   onError: (err: unknown) => void;
   maxWatchedPaths: number;
   onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void;
+  /**
+   * Reports every in-scope directory a backend encounters while planning its
+   * watch roots, so the known-kinds index can be built in that same pass.
+   */
+  onDirectorySeen?: (absolutePath: string) => void;
 }
 
 export type TreeWatchBackendFactory = (
@@ -476,88 +492,344 @@ function startChokidarTreeWatch(
 }
 
 /**
- * Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
+ * macOS/Windows backend: recursive `fs.watch` handles placed over the in-scope
+ * tree ONLY, as planned by {@link planWatchRoots}.
  *
- * Backend selection (the whole point of this helper):
- *   - macOS / Windows → ONE recursive `fs.watch` handle for the entire tree.
- *     chokidar 4 dropped its `fsevents` backend, so on macOS it watches via
- *     kqueue, which costs ~1 open fd PER watched path — ~11k fds over a real
- *     HQ tree, which EMFILEs under the default soft `ulimit -n` (256) and
- *     silently kills the watcher (instant sync then falls back to the poll).
- *     A single recursive watch is 1 handle regardless of tree size.
- *   - Linux / other → chokidar (recursive `fs.watch` is unsupported there).
+ * The previous shape was a single recursive watch on hqRoot. That is one handle
+ * regardless of tree size — but the OS then reports the WHOLE tree, including
+ * the buckets sync never uploads. On a real HQ root that is `repos/` (~26k
+ * directories) and `workspace/worktrees/` (~62k), where agent builds, installs,
+ * and git checkouts churn constantly. Each of those events woke the runner and
+ * was dropped only after the emit filter ran, which was enough allocation churn
+ * to pin the process above 100% CPU in GC.
  *
- * The recursive backend does NO descent-time filtering (it can't — the OS
- * reports the whole subtree), so EVERY change is funneled to `onEvent`, which
- * re-applies the emit filter via {@link TreeWatcher.handleEvent}. Excluded
- * paths (`repos/`, `node_modules/`, out-of-scope companies) are cheaply
- * dropped there before they arm the debounce. This is also why the recursive
- * backend is immune to the `.hqinclude` ancestor-descent pruning that the
- * chokidar backend needs {@link toChokidarIgnored} to avoid.
+ * So instead: plan the minimum cover of the uploadable tree and watch that.
+ * A directory whose entire subtree is in scope takes one recursive watch; a
+ * directory containing an excluded child is watched non-recursively and its
+ * in-scope children are planned separately. Excluded subtrees get no watch at
+ * all, so the OS never reports them.
+ *
+ * Handle count stays small (tens, not thousands — well clear of the kqueue fd
+ * exhaustion that made chokidar 4 unusable here), and the emit filter still
+ * runs on every delivered event, so a depth-bounded plan admitting some residue
+ * is a performance concern, never a correctness one.
  */
-function startTreeWatch(
+/**
+ * How long the temporary whole-tree bootstrap watch is held open after the
+ * scoped plan goes live, so events the OS observed during the planning walk
+ * still drain instead of being dropped with the handle.
+ */
+const BOOTSTRAP_DRAIN_MS = 2000;
+
+export interface ScopedWatchBackend extends WatchBackend {
+  /**
+   * Planned watches that could not be attached during the initial plan. Any
+   * failure here means the scoped plan has a permanent hole, so the caller
+   * discards it and falls back to chokidar rather than running with partial
+   * coverage.
+   */
+  attachFailures: number;
+}
+
+export function startScopedRecursiveWatch(
   hqRoot: string,
   shouldEmit: WatchPathFilter,
   onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
   onError: (err: unknown) => void,
-  maxWatchedPaths: number,
-  onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
-): WatchBackend {
-  if (supportsRecursiveWatch()) {
+  onDirectorySeen?: (absolutePath: string) => void,
+): ScopedWatchBackend {
+  const root = path.resolve(hqRoot);
+  const watchers = new Map<string, fs.FSWatcher>();
+  const recursiveRoots = new Set<string>();
+  const recursiveFlag = new Map<string, boolean>();
+  let closed = false;
+  let attachFailures = 0;
+  /** Retires the bootstrap watch — replaced once that watch exists. */
+  let closeBootstrap: () => void = () => {};
+
+  /** lstat that answers "is this a directory right now", never throwing. */
+  function isDirectoryNow(absolutePath: string): boolean {
     try {
-      const native = fs.watch(
-        hqRoot,
-        { recursive: true, persistent: true },
+      return fs.lstatSync(absolutePath).isDirectory();
+    } catch {
+      return false;
+    }
+  }
+
+  function exists(absolutePath: string): boolean {
+    try {
+      fs.lstatSync(absolutePath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Release the watch on `dir` AND on everything beneath it.
+   *
+   * When a directory is removed the OS reports that directory, not each of its
+   * descendants — so dropping only the exact path leaks every watch underneath
+   * it. Those handles stay open forever, and a leaked recursive root keeps
+   * marking its (now absent) subtree as covered, so a later recreation is never
+   * re-attached. These runners live for many hours, so the leak accumulates
+   * across every directory the tree churns through.
+   */
+  function dropWatch(dir: string): void {
+    const prefix = dir + path.sep;
+    for (const watched of [...watchers.keys()]) {
+      if (watched !== dir && !watched.startsWith(prefix)) continue;
+      const handle = watchers.get(watched);
+      if (handle) {
+        try {
+          handle.close();
+        } catch {
+          /* already closed */
+        }
+      }
+      watchers.delete(watched);
+      recursiveRoots.delete(watched);
+      recursiveFlag.delete(watched);
+    }
+  }
+
+  function addWatch(dir: string, recursive: boolean): void {
+    if (closed || watchers.has(dir)) return;
+    let handle: fs.FSWatcher;
+    try {
+      handle = fs.watch(
+        dir,
+        { recursive, persistent: true },
         (eventType, filename) => {
-          // `filename` is relative to hqRoot (or null/empty if the platform
-          // couldn't provide it — nothing actionable then). `String()` coerces
-          // both the string and (older @types/node) Buffer shapes.
+          // `filename` is relative to THIS watch's directory (or null/empty if
+          // the platform couldn't provide it — nothing actionable then).
+          // `String()` coerces both the string and (older @types/node) Buffer
+          // shapes.
           if (filename == null) return;
           const rel = String(filename);
           if (rel === "") return;
-          onEvent(
-            path.resolve(hqRoot, rel),
-            eventType === "change" ? "change" : "rename",
-          );
+          const abs = path.resolve(dir, rel);
+          const kind: BackendChangeKind =
+            eventType === "change" ? "change" : "rename";
+          if (kind === "rename") reconcileCoverage(abs);
+          onEvent(abs, kind);
         },
       );
-      let closed = false;
-      let fallback: WatchBackend | null = null;
-      native.on("error", (err) => {
-        onError(err);
-        if (closed || fallback !== null) return;
+    } catch (err) {
+      // A planned watch we could not attach. Report it and record the hole so
+      // the caller can reject the whole plan instead of running half-blind.
+      attachFailures += 1;
+      onError(err);
+      return;
+    }
+    handle.on("error", (err) => {
+      onError(err);
+      if (closed) return;
+      // Rebuild this root rather than leaving it permanently unwatched. The
+      // onError above has already asked TreeWatcher for a reconcile, so a gap
+      // between the drop and the re-attach cannot lose data.
+      const wasRecursive = recursiveFlag.get(dir) ?? recursive;
+      dropWatch(dir);
+      if (exists(dir) && shouldEmit(dir, true)) addWatch(dir, wasRecursive);
+    });
+    watchers.set(dir, handle);
+    recursiveFlag.set(dir, recursive);
+    if (recursive) recursiveRoots.add(dir);
+  }
+
+  function applyPlan(plan: WatchRootPlan): void {
+    for (const dir of plan.recursive) addWatch(dir, true);
+    for (const dir of plan.shallow) addWatch(dir, false);
+  }
+
+  function planAndAttach(dir: string): void {
+    applyPlan(planWatchRoots(dir, shouldEmit, { onDirectory: onDirectorySeen }));
+  }
+
+  /**
+   * Keep the live watch set honest about a path that just appeared, moved, or
+   * vanished. Three cases, all of which silently break coverage if ignored:
+   *
+   *  1. A watched directory was deleted (or deleted and recreated, which gives
+   *     the replacement a new inode). Its stale handle would otherwise make the
+   *     path look covered forever, so drop it and let the recreate re-plan.
+   *  2. An EXCLUDED directory was created under a recursive root — e.g.
+   *     `workspace/worktrees/` or a fresh `node_modules/` that did not exist at
+   *     plan time. The root now covers a bucket we must never watch, which is
+   *     exactly the firehose this backend exists to prevent, so re-plan it.
+   *  3. An in-scope directory was created under a SHALLOW watch, so nothing
+   *     covers its subtree yet. Attach before the event is handed on.
+   */
+  function reconcileCoverage(absolutePath: string): void {
+    if (closed) return;
+
+    // (1) stale handle for a path that no longer exists.
+    if (watchers.has(absolutePath) && !exists(absolutePath)) {
+      dropWatch(absolutePath);
+      return;
+    }
+
+    const inScope = shouldEmit(absolutePath, true);
+    const covered = isCoveredByRecursiveRoot(absolutePath, recursiveRoots);
+
+    // (2) newly-created exclusion swallowed by an existing recursive root. The
+    // lstat is paid only for excluded paths under a recursive root — rare, and
+    // strictly cheaper than the unconditional stat this backend replaced.
+    if (!inScope) {
+      if (covered && isDirectoryNow(absolutePath)) {
+        for (const rootDir of [...recursiveRoots]) {
+          if (
+            absolutePath === rootDir ||
+            absolutePath.startsWith(rootDir + path.sep)
+          ) {
+            dropWatch(rootDir);
+            planAndAttach(rootDir);
+          }
+        }
+      }
+      return;
+    }
+
+    // (3) in-scope directory with no coverage yet.
+    if (covered || watchers.has(absolutePath)) return;
+    if (!isDirectoryNow(absolutePath)) return;
+    onDirectorySeen?.(absolutePath);
+    planAndAttach(absolutePath);
+  }
+
+  // Cover the whole tree BEFORE planning. The plan walk below is synchronous
+  // and takes seconds on a real HQ root; without this, every change during that
+  // window would be invisible to event-driven push and would wait for the next
+  // cadence poll. This temporary watch is exactly the old backend's shape, so
+  // the bootstrap window is no worse than the previous behavior — and it is
+  // dropped the moment the scoped plan is live.
+  let bootstrap: fs.FSWatcher | null = null;
+  try {
+    bootstrap = fs.watch(
+      root,
+      { recursive: true, persistent: true },
+      (eventType, filename) => {
+        if (filename == null) return;
+        const rel = String(filename);
+        if (rel === "") return;
+        onEvent(
+          path.resolve(root, rel),
+          eventType === "change" ? "change" : "rename",
+        );
+      },
+    );
+    bootstrap.on("error", onError);
+  } catch (err) {
+    // Recursive watch is unavailable at all — the plan below cannot work
+    // either, so record the hole and let the caller fall back.
+    attachFailures += 1;
+    onError(err);
+  }
+
+  // Unbounded on purpose: this walk replaces TreeWatcher's separate known-kinds
+  // seed walk (same tree, same filter, ~3.6s on a real HQ root), so planning
+  // the tree exhaustively costs nothing over the status quo and leaves no
+  // excluded subtree covered by a recursive root.
+  planAndAttach(root);
+
+  // The scoped plan is live, so narrow from whole-tree to in-scope-only — but
+  // NOT synchronously. Events observed during the planning walk can still be
+  // sitting in the OS queue undelivered, and closing the handle here drops
+  // them, reopening the very gap the bootstrap watch exists to close. Hold it
+  // open for one drain window instead; the overlap only costs duplicate events
+  // for excluded paths, which the emit filter drops and the debounce coalesces.
+  if (bootstrap) {
+    const handle = bootstrap;
+    bootstrap = null;
+    const timer = setTimeout(() => {
+      try {
+        handle.close();
+      } catch {
+        /* already closed */
+      }
+      closeBootstrap = () => {};
+    }, BOOTSTRAP_DRAIN_MS);
+    // Never hold the process open just to retire a temporary watch.
+    (timer as { unref?: () => void }).unref?.();
+    closeBootstrap = () => {
+      clearTimeout(timer);
+      try {
+        handle.close();
+      } catch {
+        /* already closed */
+      }
+      closeBootstrap = () => {};
+    };
+  }
+
+  return {
+    close: () => {
+      closed = true;
+      closeBootstrap();
+      for (const handle of watchers.values()) {
         try {
-          native.close();
+          handle.close();
         } catch {
           /* already closed */
         }
-        try {
-          fallback = startChokidarTreeWatch(
-            hqRoot,
-            shouldEmit,
-            onEvent,
-            onError,
-            maxWatchedPaths,
-            onWatchBudgetExceeded,
-          );
-        } catch (fallbackErr) {
-          onError(fallbackErr);
-        }
-      });
-      return {
-        close: () => {
-          closed = true;
-          try {
-            native.close();
-          } catch {
-            /* already closed */
-          }
-          fallback?.close();
-          fallback = null;
-        },
-        needsKnownKinds: true,
-        watchedPathCount: () => fallback?.watchedPathCount() ?? 1,
-      };
+      }
+      watchers.clear();
+      recursiveRoots.clear();
+      recursiveFlag.clear();
+    },
+    needsKnownKinds: true,
+    seededKnownKinds: onDirectorySeen !== undefined,
+    watchedPathCount: () => watchers.size,
+    attachFailures,
+  };
+}
+
+/**
+ * Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
+ *
+ * Backend selection (the whole point of this helper):
+ *   - macOS / Windows → scoped recursive `fs.watch` handles over the in-scope
+ *     tree only ({@link startScopedRecursiveWatch}). chokidar 4 dropped its
+ *     `fsevents` backend, so on macOS it watches via kqueue, which costs ~1
+ *     open fd PER watched path — ~11k fds over a real HQ tree, which EMFILEs
+ *     under the default soft `ulimit -n` (256) and silently kills the watcher
+ *     (instant sync then falls back to the poll).
+ *   - Linux / other → chokidar (recursive `fs.watch` is unsupported there).
+ *
+ * The recursive backend does no descent-time filtering WITHIN a watched
+ * subtree, so changes there are funneled to `onEvent`, which re-applies the
+ * emit filter via {@link TreeWatcher.handleEvent}. That filter remains the
+ * authoritative gate; the watch plan is a performance optimization on top of
+ * it. This is also why the recursive backend is immune to the `.hqinclude`
+ * ancestor-descent pruning that the chokidar backend needs
+ * {@link toChokidarIgnored} to avoid.
+ */
+function startTreeWatch(
+  hqRoot: string,
+  shouldEmit: WatchPathFilter,
+  onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
+  onError: (err: unknown) => void,
+  maxWatchedPaths: number,
+  onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
+  onDirectorySeen?: (absolutePath: string) => void,
+): WatchBackend {
+  if (supportsRecursiveWatch()) {
+    try {
+      const scoped = startScopedRecursiveWatch(
+        hqRoot,
+        shouldEmit,
+        onEvent,
+        onError,
+        onDirectorySeen,
+      );
+      // Partial coverage is worse than a different backend: a planned root we
+      // could not attach is silently unwatched forever, and the operator only
+      // sees it as "instant sync stopped working for that subtree". Accept the
+      // scoped plan only when EVERY planned watch attached.
+      if (scoped.attachFailures === 0 && scoped.watchedPathCount() > 0) {
+        return scoped;
+      }
+      scoped.close();
     } catch (err) {
       // Recursive watch unexpectedly unavailable — fall back to chokidar
       // rather than leaving the daemon with no watcher at all.
@@ -889,6 +1161,8 @@ export class TreeWatcher {
         offendingPath: info.offendingPath,
         offendingTopLevel: this.topLevelForTelemetry(info.offendingPath),
       }),
+      onDirectorySeen: (absolutePath) =>
+        this.knownKinds.set(path.resolve(absolutePath), "directory"),
     };
     const backend = this.backendFactory
       ? this.backendFactory(backendOpts)
@@ -899,6 +1173,7 @@ export class TreeWatcher {
           backendOpts.onError,
           backendOpts.maxWatchedPaths,
           backendOpts.onWatchBudgetExceeded,
+          backendOpts.onDirectorySeen,
         );
     // A test backend (or a future eager backend) can discover exhaustion while
     // it is being constructed. Never retain that just-created handle after the
@@ -908,7 +1183,12 @@ export class TreeWatcher {
       return;
     }
     this.backend = backend;
-    if (backend.needsKnownKinds) this.seedKnownKinds();
+    // The scoped backend already indexed every in-scope directory while
+    // planning its watch roots — walking the tree again here would double a
+    // multi-second startup cost for an identical result.
+    if (backend.needsKnownKinds && !backend.seededKnownKinds) {
+      this.seedKnownKinds();
+    }
   }
 
   private handleBackendError(err: unknown): void {
@@ -987,6 +1267,13 @@ export class TreeWatcher {
   ): void {
     if (this.disposed) return;
     const abs = path.resolve(absolutePath);
+    // Cheap pre-gate. When a path is out of scope BOTH as a file and as a
+    // directory, no stat can change the answer — so drop it before paying the
+    // syscall below. Scoped watch roots (see planWatchRoots) keep most excluded
+    // traffic from being delivered at all; this covers the residue a
+    // depth-bounded plan still admits, at the cost of two string matches
+    // instead of one synchronous lstat.
+    if (!this.shouldEmit(abs, false) && !this.shouldEmit(abs, true)) return;
     let kind: TreeChangeKind;
     if (backendKind === "rename") {
       try {
@@ -1030,6 +1317,17 @@ export class TreeWatcher {
     }
     this.pending.set(abs, rel);
     if (kind === "unlink" || kind === "unlinkDir") {
+      // A SECOND delete event for the same path in one window must never
+      // downgrade a directory delete to a file delete. The first `unlinkDir`
+      // clears this path's known-kind hint, so a duplicate event — the OS can
+      // report one path through two overlapping watches — would re-classify it
+      // as `unlink` and, with it, narrow the deletion scope the vault applies
+      // and drop the captured descendant snapshots.
+      if (kind === "unlink" && this.pendingChanges.get(abs)?.kind === "unlinkDir") {
+        this.pending.set(abs, rel);
+        this.arm();
+        return;
+      }
       const deleteSnapshots =
         this.captureLocalDeleteSnapshots?.(rel, kind) ?? [];
       this.pendingChanges.set(abs, {