@indigoai-us/hq-cloud 6.14.26 → 6.14.28

package/src/watcher.ts

@@ -31,6 +31,28 @@ import {
 
 const DEBOUNCE_MS = 2000;
 
+/**
+ * A single Linux chokidar watcher must leave room for editors, dev servers,
+ * and other HQ runners on the same uid. This is deliberately far below the
+ * common 250k kernel maximum. The cap is on distinct directories admitted over
+ * this watcher's lifetime, which also prevents a high-churn tree from slowly
+ * accumulating an unbounded set of kernel watches.
+ */
+export const DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS = 20_000;
+export const TREE_WATCHER_MAX_WATCHED_PATHS_ENV = "HQ_SYNC_MAX_WATCHED_PATHS";
+
+function resolveMaxWatchedPaths(override: number | undefined): number {
+  const configured = override ?? Number(process.env[TREE_WATCHER_MAX_WATCHED_PATHS_ENV]);
+  if (override === undefined && process.env[TREE_WATCHER_MAX_WATCHED_PATHS_ENV] === undefined) {
+    return DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS;
+  }
+  if (Number.isFinite(configured) && configured > 0) return Math.trunc(configured);
+  console.warn(
+    `TreeWatcher ignoring invalid ${TREE_WATCHER_MAX_WATCHED_PATHS_ENV}; using ${DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS}`,
+  );
+  return DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS;
+}
+
 /**
  * Injectable clock seam (US-001).
  *
@@ -224,44 +246,146 @@ export class WatchPushDriver {
 /** Decision for a single path: emit a change for it, or ignore it. */
 export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
 
+export interface TreeWatcherWatchBudgetExceeded {
+  maxWatchedPaths: number;
+  watchedPaths: number;
+  /** Absolute directory whose admission would exceed the cap. */
+  offendingPath: string;
+}
+
+/**
+ * Tracks directories admitted to chokidar. Chokidar asks its ignored callback
+ * twice: before stat (the descent gate) and after stat (where we can safely
+ * account only for directories). This class is intentionally lifetime-bounded:
+ * a directory that is deleted and recreated cannot turn one long-running
+ * runner into an unbounded consumer.
+ */
+export class ChokidarWatchBudget {
+  private readonly admitted = new Set<string>();
+  private exhausted = false;
+
+  constructor(
+    private readonly maxWatchedPaths: number,
+    private readonly onExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
+  ) {}
+
+  admit(absolutePath: string): boolean {
+    const resolved = path.resolve(absolutePath);
+    if (this.admitted.has(resolved)) return true;
+    if (this.exhausted || this.admitted.size >= this.maxWatchedPaths) {
+      if (!this.exhausted) {
+        this.exhausted = true;
+        this.onExceeded({
+          maxWatchedPaths: this.maxWatchedPaths,
+          watchedPaths: this.admitted.size,
+          offendingPath: resolved,
+        });
+      }
+      return false;
+    }
+    this.admitted.add(resolved);
+    return true;
+  }
+
+  count(): number {
+    return this.admitted.size;
+  }
+}
+
 /**
- * Translate an emit-filter into chokidar's `ignored` predicate (true = SKIP).
+ * Translate the emit filter into chokidar's descent predicate.
  *
- * The subtlety this encodes: chokidar probes a candidate path for the descent
- * decision BEFORE it has stat()ed it, so `stats` is `undefined` on that first
- * call — and if we say "ignore" there, chokidar prunes the subtree and never
- * descends. In allowlist mode (`.hqinclude`) the in-scope leaves
- * (`companies/*\/knowledge/`) live UNDER intermediate dirs (`companies/`,
- * `companies/<slug>/`) that only survive the filter when queried as a
- * directory (the ancestor matcher fires for `isDir=true` only). With no stats
- * hint we don't know if the candidate is that ancestor dir or an in-scope
- * file, so we must NOT prune when EITHER reading would keep it. The real
- * isDir-accurate verdict is reapplied at event time (TreeWatcher.handleEvent
- * re-checks with the known kind), so this only widens descent, never emit.
+ * Chokidar first calls this with no stats, before it has chosen whether to
+ * recurse. That call is a DIRECTORY question, not an event-emission question:
+ * consulting `shouldEmit(path, false)` here lets directory-only ignores such
+ * as `node_modules/` masquerade as files and opens their whole subtree. The
+ * directory branch of createIgnoreFilter already contains the exact
+ * `.hqinclude` ancestor matcher, so it is both safe and sufficient to consult
+ * `shouldEmit(path, true)` at this gate.
  */
-function toChokidarIgnored(
+export function toChokidarIgnored(
   shouldEmit: WatchPathFilter,
   hqRoot: string,
+  budget?: ChokidarWatchBudget,
 ): (filePath: string, stats?: fs.Stats) => boolean {
   const root = path.resolve(hqRoot);
   return (filePath: string, stats?: fs.Stats): boolean => {
     // The watched root itself must NEVER be ignored, or chokidar tears down
     // the whole watch. (shouldEmit returns false for it — it's not a valid
     // emit target — so we special-case it here.)
-    if (path.resolve(filePath) === root) return false;
+    if (path.resolve(filePath) === root) {
+      return stats?.isDirectory() && budget !== undefined
+        ? !budget.admit(filePath)
+        : false;
+    }
     const isDir = stats?.isDirectory();
     if (isDir === undefined) {
-      // Pre-stat descent probe: keep the path if it could be an in-scope file
-      // OR an ancestor directory of one.
-      return !(shouldEmit(filePath, false) || shouldEmit(filePath, true));
+      // This is chokidar's descent decision. The `isDir=true` branch includes
+      // only precise .hqinclude ancestors, rather than widening descent for
+      // every path that could happen to be an allowed file.
+      return !shouldEmit(filePath, true);
+    }
+    if (isDir) {
+      return (
+        !shouldEmit(filePath, true) ||
+        (budget !== undefined && !budget.admit(filePath))
+      );
     }
     return !shouldEmit(filePath, isDir);
   };
 }
 
 /** A started watch backend. `close()` releases its OS handle(s); idempotent. */
-interface WatchBackend {
+export interface WatchBackend {
   close(): void;
+  /** Native recursive watches need rename-kind state; chokidar does not. */
+  needsKnownKinds: boolean;
+  /** Directories/OS handles admitted by this backend so degradation is auditable. */
+  watchedPathCount(): number;
+}
+
+export interface TreeWatcherDegradation {
+  reason: "watch_budget_exhausted" | "inotify_enospc";
+  maxWatchedPaths: number;
+  watchedPaths: number;
+  /** Full local path is deliberately log-only; telemetry receives its top level. */
+  offendingPath?: string;
+  offendingTopLevel?: string;
+  errorCode?: string;
+}
+
+export interface TreeWatchBackendOptions {
+  hqRoot: string;
+  shouldEmit: WatchPathFilter;
+  onEvent: (absolutePath: string, kind: BackendChangeKind) => void;
+  onError: (err: unknown) => void;
+  maxWatchedPaths: number;
+  onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void;
+}
+
+export type TreeWatchBackendFactory = (
+  opts: TreeWatchBackendOptions,
+) => WatchBackend;
+
+function isInotifyLimitError(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    "code" in err &&
+    (err as { code?: unknown }).code === "ENOSPC"
+  );
+}
+
+function errorPath(err: unknown): string | undefined {
+  if (
+    typeof err === "object" &&
+    err !== null &&
+    "path" in err &&
+    typeof (err as { path?: unknown }).path === "string"
+  ) {
+    return (err as { path: string }).path;
+  }
+  return undefined;
 }
 
 export type TreeChangeKind =
@@ -306,18 +430,36 @@ function startChokidarTreeWatch(
   shouldEmit: WatchPathFilter,
   onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
   onError: (err: unknown) => void,
+  maxWatchedPaths: number,
+  onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
 ): WatchBackend {
-  const cw = watch(hqRoot, {
-    // chokidar fallback (Linux): see toChokidarIgnored for why the descent
-    // probe must not prune ancestor dirs of allowlisted leaves.
-    ignored: toChokidarIgnored(shouldEmit, hqRoot),
-    persistent: true,
-    ignoreInitial: true,
-    awaitWriteFinish: {
-      stabilityThreshold: 500,
-      pollInterval: 100,
-    },
-  });
+  const budget = new ChokidarWatchBudget(
+    maxWatchedPaths,
+    onWatchBudgetExceeded,
+  );
+  let cw: ReturnType<typeof watch>;
+  try {
+    cw = watch(hqRoot, {
+      // On Linux chokidar adds one fs.watch/inotify watch per descended dir.
+      // The predicate therefore enforces both scope and the hard per-runner cap.
+      ignored: toChokidarIgnored(shouldEmit, hqRoot, budget),
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 500,
+        pollInterval: 100,
+      },
+    });
+  } catch (err) {
+    // Some chokidar/fs.watch versions surface ENOSPC synchronously. Route it
+    // through the same degradation path as an asynchronous backend error.
+    onError(err);
+    return {
+      close: () => {},
+      needsKnownKinds: false,
+      watchedPathCount: () => budget.count(),
+    };
+  }
   cw.on("add", (p) => onEvent(p, "add"))
     .on("change", (p) => onEvent(p, "change"))
     .on("unlink", (p) => onEvent(p, "unlink"))
@@ -328,6 +470,8 @@ function startChokidarTreeWatch(
     close: () => {
       void cw.close();
     },
+    needsKnownKinds: false,
+    watchedPathCount: () => budget.count(),
   };
 }
 
@@ -356,6 +500,8 @@ function startTreeWatch(
   shouldEmit: WatchPathFilter,
   onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
   onError: (err: unknown) => void,
+  maxWatchedPaths: number,
+  onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
 ): WatchBackend {
   if (supportsRecursiveWatch()) {
     try {
@@ -391,6 +537,8 @@ function startTreeWatch(
             shouldEmit,
             onEvent,
             onError,
+            maxWatchedPaths,
+            onWatchBudgetExceeded,
           );
         } catch (fallbackErr) {
           onError(fallbackErr);
@@ -407,6 +555,8 @@ function startTreeWatch(
           fallback?.close();
           fallback = null;
         },
+        needsKnownKinds: true,
+        watchedPathCount: () => fallback?.watchedPathCount() ?? 1,
       };
     } catch (err) {
       // Recursive watch unexpectedly unavailable — fall back to chokidar
@@ -415,7 +565,14 @@ function startTreeWatch(
     }
   }
 
-  return startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError);
+  return startChokidarTreeWatch(
+    hqRoot,
+    shouldEmit,
+    onEvent,
+    onError,
+    maxWatchedPaths,
+    onWatchBudgetExceeded,
+  );
 }
 
 /**
@@ -562,8 +719,25 @@ export interface TreeWatcherOptions {
   maxPendingPaths?: number;
   /** Approximate maximum path-string bytes retained in one debounce window. */
   maxPendingBytes?: number;
+  /**
+   * Hard cap on distinct directories admitted to the chokidar backend for the
+   * lifetime of this instance. Linux uses one inotify watch per such directory.
+   * Overrides `HQ_SYNC_MAX_WATCHED_PATHS` for callers that construct a watcher.
+   */
+  maxWatchedPaths?: number;
   /** Backlog overflow signal. Defaults to a console warning. */
   onBacklogOverflow?: (info: TreeWatcherBacklogOverflow) => void;
+  /**
+   * Structured degradation/metric seam. It fires exactly once when the watch
+   * cap or the kernel's inotify limit disables event watching; cadence polling
+   * remains the correctness path.
+   */
+  onDegraded?: (info: TreeWatcherDegradation) => void;
+  /** Optional cloud telemetry sink for the same degradation signal. */
+  telemetryClient?: CloudTelemetryClient | null;
+  telemetryClaims?: TelemetryClaims | null;
+  /** Test seam for a watch backend; production uses native/chokidar selection. */
+  backendFactory?: TreeWatchBackendFactory;
   captureLocalDeleteSnapshots?: (
     relativePath: string,
     kind: "unlink" | "unlinkDir",
@@ -622,7 +796,12 @@ export class TreeWatcher {
   private readonly shouldEmit: WatchPathFilter;
   private readonly maxPendingPaths: number;
   private readonly maxPendingBytes: number;
+  private readonly maxWatchedPaths: number;
   private readonly onBacklogOverflow: (info: TreeWatcherBacklogOverflow) => void;
+  private readonly onDegraded: (info: TreeWatcherDegradation) => void;
+  private readonly telemetryClient: CloudTelemetryClient | null | undefined;
+  private readonly telemetryClaims: TelemetryClaims | null | undefined;
+  private readonly backendFactory?: TreeWatchBackendFactory;
   private readonly captureLocalDeleteSnapshots?: TreeWatcherOptions["captureLocalDeleteSnapshots"];
   private backend: WatchBackend | null = null;
   private timer: unknown = null;
@@ -636,6 +815,7 @@ export class TreeWatcher {
   private overflowLogged = false;
   private droppedPaths = 0;
   private droppedBytes = 0;
+  private degraded = false;
   private disposed = false;
 
   constructor(opts: TreeWatcherOptions) {
@@ -652,6 +832,7 @@ export class TreeWatcher {
       1,
       opts.maxPendingBytes ?? DEFAULT_TREE_WATCHER_MAX_PENDING_BYTES,
     );
+    this.maxWatchedPaths = resolveMaxWatchedPaths(opts.maxWatchedPaths);
     this.onBacklogOverflow =
       opts.onBacklogOverflow ??
       ((info) => {
@@ -659,6 +840,16 @@ export class TreeWatcher {
           `TreeWatcher backlog cap exceeded; dropping ${info.droppedPaths} path(s) until the next resync`,
         );
       });
+    this.onDegraded =
+      opts.onDegraded ??
+      ((info) => {
+        console.error(
+          JSON.stringify({ event: "watcher.degraded", ...info }),
+        );
+      });
+    this.telemetryClient = opts.telemetryClient;
+    this.telemetryClaims = opts.telemetryClaims;
+    this.backendFactory = opts.backendFactory;
     this.captureLocalDeleteSnapshots = opts.captureLocalDeleteSnapshots;
   }
 
@@ -684,17 +875,98 @@ export class TreeWatcher {
    * for why per-path watching is avoided (kqueue fd exhaustion → EMFILE).
    */
   start(): void {
-    if (this.disposed || this.backend) return;
-    this.backend = startTreeWatch(
-      this.hqRoot,
-      this.shouldEmit,
-      (absolutePath, kind) => this.handleEvent(absolutePath, kind),
-      (err) => {
-        console.error("TreeWatcher error:", err);
-        this.signalBackendResync();
+    if (this.disposed || this.degraded || this.backend) return;
+    const backendOpts: TreeWatchBackendOptions = {
+      hqRoot: this.hqRoot,
+      shouldEmit: this.shouldEmit,
+      onEvent: (absolutePath, kind) => this.handleEvent(absolutePath, kind),
+      onError: (err) => this.handleBackendError(err),
+      maxWatchedPaths: this.maxWatchedPaths,
+      onWatchBudgetExceeded: (info) => this.degrade({
+        reason: "watch_budget_exhausted",
+        maxWatchedPaths: info.maxWatchedPaths,
+        watchedPaths: info.watchedPaths,
+        offendingPath: info.offendingPath,
+        offendingTopLevel: this.topLevelForTelemetry(info.offendingPath),
+      }),
+    };
+    const backend = this.backendFactory
+      ? this.backendFactory(backendOpts)
+      : startTreeWatch(
+          backendOpts.hqRoot,
+          backendOpts.shouldEmit,
+          backendOpts.onEvent,
+          backendOpts.onError,
+          backendOpts.maxWatchedPaths,
+          backendOpts.onWatchBudgetExceeded,
+        );
+    // A test backend (or a future eager backend) can discover exhaustion while
+    // it is being constructed. Never retain that just-created handle after the
+    // degradation path has deliberately disabled event watching.
+    if (this.degraded) {
+      backend.close();
+      return;
+    }
+    this.backend = backend;
+    if (backend.needsKnownKinds) this.seedKnownKinds();
+  }
+
+  private handleBackendError(err: unknown): void {
+    if (isInotifyLimitError(err)) {
+      const offendingPath = errorPath(err);
+      this.degrade({
+        reason: "inotify_enospc",
+        maxWatchedPaths: this.maxWatchedPaths,
+        watchedPaths: this.backend?.watchedPathCount() ?? 0,
+        offendingPath,
+        offendingTopLevel: offendingPath
+          ? this.topLevelForTelemetry(offendingPath)
+          : undefined,
+        errorCode: "ENOSPC",
+      });
+      return;
+    }
+    console.error("TreeWatcher error:", err);
+    this.signalBackendResync();
+  }
+
+  private topLevelForTelemetry(absolutePath: string): string | undefined {
+    const rel = path.relative(this.hqRoot, absolutePath).split(path.sep);
+    const topLevel = rel[0];
+    return topLevel && topLevel !== ".." ? topLevel : undefined;
+  }
+
+  private degrade(info: TreeWatcherDegradation): void {
+    if (this.disposed || this.degraded) return;
+    this.degraded = true;
+    // Closing synchronously prevents further directory admissions while
+    // chokidar's initial walk is still in flight.
+    this.backend?.close();
+    this.backend = null;
+    try {
+      this.onDegraded(info);
+    } catch (err) {
+      console.error("TreeWatcher degradation reporter error:", err);
+    }
+    void emitCloudTelemetry(
+      this.telemetryClient,
+      {
+        eventName: "watcher_degraded",
+        source: "watcher",
+        properties: {
+          reason: info.reason,
+          maxWatchedPaths: info.maxWatchedPaths,
+          watchedPaths: info.watchedPaths,
+          ...(info.offendingTopLevel
+            ? { offendingTopLevel: info.offendingTopLevel }
+            : {}),
+        },
       },
+      { claims: this.telemetryClaims },
     );
-    this.seedKnownKinds();
+    // The existing overflow/resync batch causes the runner to use its scoped
+    // drain while the cadence poll continues covering everything else.
+    this.signalBackendResync();
   }
 
   private signalBackendResync(): void {
@@ -767,7 +1039,12 @@ export class TreeWatcher {
       this.knownKinds.delete(abs);
     } else {
       this.pendingChanges.set(abs, { kind });
-      this.knownKinds.set(abs, isDir ? "directory" : "file");
+      // Only directories are needed to disambiguate a native recursive
+      // rename-after-delete. Unknown paths already correctly default to an
+      // unlink file, so retaining every file ever touched made this map grow
+      // without bound in a long-lived runner.
+      if (isDir) this.knownKinds.set(abs, "directory");
+      else this.knownKinds.delete(abs);
     }
     this.arm();
   }
@@ -786,8 +1063,6 @@ export class TreeWatcher {
           if (!this.shouldEmit(absolutePath, true)) continue;
           this.knownKinds.set(path.resolve(absolutePath), "directory");
           visit(absolutePath);
-        } else if (this.shouldEmit(absolutePath, false)) {
-          this.knownKinds.set(path.resolve(absolutePath), "file");
         }
       }
     };
@@ -872,6 +1147,11 @@ export class TreeWatcher {
     return this.timer === null ? 0 : 1;
   }
 
+  /** Number of native-rename directory hints retained for deletion handling. */
+  knownDirectoryCount(): number {
+    return this.knownKinds.size;
+  }
+
   /**
    * Stop watching: close the watch backend (releasing its OS handle) and
    * cancel any pending debounce timer. Idempotent. The instance can be