@indigoai-us/hq-cloud 5.27.0 → 5.29.0

package/src/watcher.ts

@@ -381,6 +381,96 @@ function toChokidarIgnored(
   };
 }
 
+/** A started watch backend. `close()` releases its OS handle(s); idempotent. */
+interface WatchBackend {
+  close(): void;
+}
+
+/**
+ * Platforms where Node's `fs.watch(dir, { recursive: true })` is natively
+ * supported by a SINGLE OS handle: macOS (FSEvents) and Windows
+ * (ReadDirectoryChangesW). On Linux, recursive `fs.watch` is NOT implemented
+ * (it throws `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM`), so we fall back to
+ * chokidar there.
+ */
+function supportsRecursiveWatch(): boolean {
+  return process.platform === "darwin" || process.platform === "win32";
+}
+
+/**
+ * Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
+ *
+ * Backend selection (the whole point of this helper):
+ *   - macOS / Windows → ONE recursive `fs.watch` handle for the entire tree.
+ *     chokidar 4 dropped its `fsevents` backend, so on macOS it watches via
+ *     kqueue, which costs ~1 open fd PER watched path — ~11k fds over a real
+ *     HQ tree, which EMFILEs under the default soft `ulimit -n` (256) and
+ *     silently kills the watcher (instant sync then falls back to the poll).
+ *     A single recursive watch is 1 handle regardless of tree size.
+ *   - Linux / other → chokidar (recursive `fs.watch` is unsupported there).
+ *
+ * The recursive backend does NO descent-time filtering (it can't — the OS
+ * reports the whole subtree), so EVERY change is funneled to `onEvent`, which
+ * re-applies the emit filter via {@link TreeWatcher.handleEvent}. Excluded
+ * paths (`repos/`, `node_modules/`, out-of-scope companies) are cheaply
+ * dropped there before they arm the debounce. This is also why the recursive
+ * backend is immune to the `.hqinclude` ancestor-descent pruning that the
+ * chokidar backend needs {@link toChokidarIgnored} to avoid.
+ */
+function startTreeWatch(
+  hqRoot: string,
+  shouldEmit: WatchPathFilter,
+  onEvent: (absolutePath: string) => void,
+  onError: (err: unknown) => void,
+): WatchBackend {
+  if (supportsRecursiveWatch()) {
+    try {
+      const native = fs.watch(
+        hqRoot,
+        { recursive: true, persistent: true },
+        (_eventType, filename) => {
+          // `filename` is relative to hqRoot (or null/empty if the platform
+          // couldn't provide it — nothing actionable then). `String()` coerces
+          // both the string and (older @types/node) Buffer shapes.
+          if (filename == null) return;
+          const rel = String(filename);
+          if (rel === "") return;
+          onEvent(path.resolve(hqRoot, rel));
+        },
+      );
+      native.on("error", onError);
+      return { close: () => native.close() };
+    } catch (err) {
+      // Recursive watch unexpectedly unavailable — fall back to chokidar
+      // rather than leaving the daemon with no watcher at all.
+      onError(err);
+    }
+  }
+
+  const cw = watch(hqRoot, {
+    // chokidar fallback (Linux): see toChokidarIgnored for why the descent
+    // probe must not prune ancestor dirs of allowlisted leaves.
+    ignored: toChokidarIgnored(shouldEmit, hqRoot),
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 500,
+      pollInterval: 100,
+    },
+  });
+  cw.on("add", onEvent)
+    .on("change", onEvent)
+    .on("unlink", onEvent)
+    .on("addDir", onEvent)
+    .on("unlinkDir", onEvent)
+    .on("error", onError);
+  return {
+    close: () => {
+      void cw.close();
+    },
+  };
+}
+
 /**
  * Build the composite emit-decision predicate. Returns true when a change to
  * `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
@@ -478,7 +568,7 @@ export class TreeWatcher {
   private readonly debounceMs: number;
   private readonly clock: Clock;
   private readonly shouldEmit: WatchPathFilter;
-  private watcher: FSWatcher | null = null;
+  private backend: WatchBackend | null = null;
   private timer: unknown = null;
   private listeners = new Set<TreeChangeListener>();
   /** Paths accumulated for the current (in-flight) debounce window. */
@@ -508,48 +598,33 @@ export class TreeWatcher {
 
   /**
    * Begin watching. Idempotent — a second call while already running is a
-   * no-op (no second chokidar instance, no leaked fds).
+   * no-op (no second watch backend, no leaked handles).
+   *
+   * Uses a SINGLE recursive `fs.watch` on macOS/Windows (1 OS handle for the
+   * whole tree) and falls back to chokidar on Linux. See {@link startTreeWatch}
+   * for why per-path watching is avoided (kqueue fd exhaustion → EMFILE).
    */
   start(): void {
-    if (this.disposed || this.watcher) return;
-
-    this.watcher = watch(this.hqRoot, {
-      // `ignored` returns true to SKIP. chokidar probes a candidate for the
-      // descent decision BEFORE stat()ing it (stats undefined on that call),
-      // so a naive `!shouldEmit(path, stats?.isDirectory())` treats the
-      // statless probe as a file and prunes intermediate dirs (`companies/`)
-      // that only survive the allowlist filter as directories — starving the
-      // in-scope leaves below them. toChokidarIgnored keeps a statless probe
-      // when EITHER the file or the directory reading would; the accurate
-      // isDir verdict is reapplied at event time in handleEvent.
-      ignored: toChokidarIgnored(this.shouldEmit, this.hqRoot),
-      persistent: true,
-      ignoreInitial: true,
-      awaitWriteFinish: {
-        stabilityThreshold: 500,
-        pollInterval: 100,
-      },
-    });
-
-    const onEvent = (absolutePath: string) => this.handleEvent(absolutePath);
-    this.watcher
-      .on("add", onEvent)
-      .on("change", onEvent)
-      .on("unlink", onEvent)
-      .on("addDir", onEvent)
-      .on("unlinkDir", onEvent)
-      .on("error", (err) => console.error("TreeWatcher error:", err));
+    if (this.disposed || this.backend) return;
+    this.backend = startTreeWatch(
+      this.hqRoot,
+      this.shouldEmit,
+      (absolutePath) => this.handleEvent(absolutePath),
+      (err) => console.error("TreeWatcher error:", err),
+    );
   }
 
   /**
-   * Test/seam entry point: feed a raw filesystem path as if chokidar reported
-   * it. Applies the emit filter then arms the debounce. Real chokidar events
-   * route through here too.
+   * Test/seam entry point: feed a raw filesystem path as if the backend
+   * reported it. Applies the emit filter then arms the debounce. Real watch
+   * events route through here too — and for the recursive backend this is the
+   * ONLY place filtering happens, so out-of-scope paths are dropped here.
    */
   handleEvent(absolutePath: string): void {
     if (this.disposed) return;
-    // Defensive: chokidar's `ignored` already filtered, but a path that slips
-    // through (or a synthetic test event) is re-checked here.
+    // The recursive backend does no descent-time filtering, so this is the
+    // authoritative emit gate; the chokidar backend's `ignored` pre-filters
+    // but a slipped-through (or synthetic test) path is re-checked here too.
     if (!this.shouldEmit(absolutePath, false)) return;
     const abs = path.resolve(absolutePath);
     const rel = path.relative(this.hqRoot, abs).split(path.sep).join("/");
@@ -587,9 +662,9 @@ export class TreeWatcher {
     }
   }
 
-  /** True while the chokidar watcher is active. */
+  /** True while the watch backend is active. */
   isWatching(): boolean {
-    return this.watcher !== null;
+    return this.backend !== null;
   }
 
   /** Number of pending debounce timers — a leak check for tests. */
@@ -598,9 +673,9 @@ export class TreeWatcher {
   }
 
   /**
-   * Stop watching: close the chokidar watcher (releasing fds) and cancel any
-   * pending debounce timer. Idempotent. The instance can be restarted with
-   * {@link start} unless {@link dispose} was called.
+   * Stop watching: close the watch backend (releasing its OS handle) and
+   * cancel any pending debounce timer. Idempotent. The instance can be
+   * restarted with {@link start} unless {@link dispose} was called.
    */
   stop(): void {
     if (this.timer !== null) {
@@ -608,9 +683,9 @@ export class TreeWatcher {
       this.timer = null;
     }
     this.pending.clear();
-    if (this.watcher) {
-      void this.watcher.close();
-      this.watcher = null;
+    if (this.backend) {
+      this.backend.close();
+      this.backend = null;
     }
   }
 

package/test/e2e/watcher-recursive-backend.test.ts

@@ -0,0 +1,115 @@
+/**
+ * REAL-fs regression E2E for the single-recursive-watch backend.
+ *
+ * Why this exists:
+ * chokidar 4 dropped its `fsevents` backend, so on macOS it watches via kqueue
+ * — ~1 open fd PER watched path. Over a real HQ tree (~11k files+dirs) that
+ * exhausts the default soft `ulimit -n` (256) with EMFILE, silently killing the
+ * watcher so instant sync degrades to the 10-min poll. The fix replaces the
+ * per-path chokidar watch with a SINGLE recursive `fs.watch` on macOS/Windows
+ * (1 OS handle for the whole tree), filtering events per-path at emit time.
+ *
+ * These tests pin the two load-bearing behaviors of that backend (both also
+ * hold for the chokidar Linux fallback, so the suite is backend-agnostic):
+ *   1. A file created under a subtree that DID NOT EXIST when the watcher
+ *      started still fires — i.e. coverage doesn't depend on per-directory
+ *      watch registration at start time.
+ *   2. An out-of-scope edit (`repos/`, excluded by DEFAULT_IGNORES) does NOT
+ *      fire — the OS reports it under a recursive watch, and it must be dropped
+ *      by the emit filter, not woken on.
+ */
+
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { TreeWatcher } from "../../src/watcher.js";
+
+let hqRoot: string;
+let watcher: TreeWatcher | null = null;
+
+beforeEach(async () => {
+  hqRoot = await mkdtemp(path.join(tmpdir(), "hqcloud-recwatch-e2e-"));
+  await writeFile(path.join(hqRoot, ".hqinclude"), "companies/*/knowledge/\n");
+  // Only the `acme` company exists at start; `newco` is created later to prove
+  // the recursive watch covers subtrees born after start().
+  await mkdir(path.join(hqRoot, "companies", "acme", "knowledge"), {
+    recursive: true,
+  });
+});
+
+afterEach(async () => {
+  watcher?.dispose();
+  watcher = null;
+  await rm(hqRoot, { recursive: true, force: true });
+});
+
+/** Resolve true if the watcher emits a debounced change within `ms`, else false. */
+function waitForEmit(w: TreeWatcher, ms: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const off = w.onChange(() => {
+      if (settled) return;
+      settled = true;
+      off();
+      clearTimeout(t);
+      resolve(true);
+    });
+    const t = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      off();
+      resolve(false);
+    }, ms);
+  });
+}
+
+describe("event-push watcher -- single recursive watch backend", () => {
+  it(
+    "FIRES for an in-scope file created under a subtree that did not exist at start",
+    async () => {
+      watcher = new TreeWatcher({ hqRoot, debounceMs: 250, personalMode: false });
+      watcher.start();
+      // Let the recursive watch establish before any mutation.
+      await new Promise((r) => setTimeout(r, 600));
+
+      // Create a brand-new company subtree AFTER the watcher started. A
+      // per-directory watcher would only catch this if it dynamically
+      // registered the new dirs; the recursive watch covers it inherently.
+      await mkdir(path.join(hqRoot, "companies", "newco", "knowledge"), {
+        recursive: true,
+      });
+
+      const emitted = waitForEmit(watcher, 4000);
+      await writeFile(
+        path.join(hqRoot, "companies", "newco", "knowledge", "fresh.md"),
+        "# fresh\n",
+      );
+      expect(await emitted).toBe(true);
+    },
+    8000,
+  );
+
+  it(
+    "does NOT fire for an out-of-scope edit the OS reports under the recursive watch (repos/)",
+    async () => {
+      watcher = new TreeWatcher({ hqRoot, debounceMs: 250, personalMode: false });
+      watcher.start();
+      await new Promise((r) => setTimeout(r, 600));
+
+      // `repos/` is excluded by DEFAULT_IGNORES. Under a recursive OS watch the
+      // event is still delivered, so the emit filter — not watch-time pruning —
+      // is what must drop it.
+      await mkdir(path.join(hqRoot, "repos", "some-repo"), { recursive: true });
+      const emitted = waitForEmit(watcher, 2500);
+      await writeFile(
+        path.join(hqRoot, "repos", "some-repo", "code.ts"),
+        "export const x = 1;\n",
+      );
+      expect(await emitted).toBe(false);
+    },
+    8000,
+  );
+});