@indigoai-us/hq-cloud 6.18.89 → 6.18.90

package/dist/watcher.js

@@ -292,6 +292,9 @@ export class ChokidarWatchBudget {
         this.admitted.add(resolved);
         return true;
     }
+    paths() {
+        return [...this.admitted];
+    }
     count() {
         return this.admitted.size;
     }
@@ -422,6 +425,7 @@ function startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatched
             mode: forcePolling ? "polling" : "chokidar",
             needsKnownKinds: false,
             watchedPathCount: () => budget.count(),
+            watchedDirectories: () => budget.paths().map((watched) => ({ path: watched, recursive: false })),
             probeCoverage,
         };
     }
@@ -444,6 +448,7 @@ function startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatched
         mode: forcePolling ? "polling" : "chokidar",
         needsKnownKinds: false,
         watchedPathCount: () => budget.count(),
+        watchedDirectories: () => budget.paths().map((watched) => ({ path: watched, recursive: false })),
         probeCoverage,
         onReady: (listener) => {
             if (ready) {
@@ -484,6 +489,16 @@ function startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatched
  * still drain instead of being dropped with the handle.
  */
 const BOOTSTRAP_DRAIN_MS = 2000;
+/**
+ * How many directory levels below the HQ root the planning bootstrap splits
+ * around excluded directories (worktrees, node_modules, nested .git, and
+ * buckets the filter rejects) before it falls back to one recursive watch.
+ * This covers the known worktree roots and the top of each company's project
+ * folders. An excluded directory deeper than this can sit under a bootstrap
+ * watch for the few seconds planning takes; the emit filter drops its events
+ * and the scoped plan that replaces the bootstrap never covers it.
+ */
+const BOOTSTRAP_SPLIT_DEPTH = 6;
 /**
  * How long after the recursive watch set changes (a re-plan, or a new
  * directory attached) the backend waits before sweeping for files it may have
@@ -973,27 +988,35 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             sweepAfterSettle(rootDir, startedAt - WATCH_SET_SWEEP_LOOKBACK_MS);
         });
     }
-    // Cover the whole tree BEFORE planning. The plan walk below is synchronous
-    // and takes seconds on a real HQ root; without this, every change during that
+    // Cover the tree BEFORE planning. The plan walk below is synchronous and
+    // takes seconds on a real HQ root; without this, every change during that
     // window would be invisible to event-driven push and would wait for the next
-    // cadence poll. This temporary watch is exactly the old backend's shape, so
-    // the bootstrap window is no worse than the previous behavior — and it is
-    // dropped the moment the scoped plan is live.
+    // cadence poll. These temporary watches are dropped once the scoped plan is
+    // live.
+    // They used to be one recursive watch on the root. That made the OS report
+    // every worktree checkout on the box while planning ran (on one Windows host,
+    // 157k files under a top-level `worktrees/`), so the bootstrap now splits
+    // around excluded directories the same way the plan does, only shallower: a
+    // directory with no excluded child gets one recursive watch, otherwise it
+    // gets a shallow watch and its in-scope children are split in turn, down to
+    // BOOTSTRAP_SPLIT_DEPTH. Below that depth one recursive watch is taken and
+    // the emit filter drops the excluded events, as before.
     // No such watch exists in shallow mode: Node's Linux "recursive" shim would
     // walk the WHOLE tree — worktrees, node_modules and all — to emulate it.
     // In shallow mode, each parent is attached before its child list is read.
-    let bootstrap = null;
-    if (recursiveWatches) {
+    const bootstrap = [];
+    const openBootstrap = (dir, recursive) => {
         try {
-            bootstrap = fs.watch(root, { recursive: true, persistent: true }, (eventType, filename) => {
+            const handle = (opts.watchDirectory ?? fs.watch)(dir, { recursive, persistent: true }, (eventType, filename) => {
                 if (filename == null)
                     return;
                 const rel = String(filename);
                 if (rel === "")
                     return;
-                onEvent(path.resolve(root, rel), eventType === "change" ? "change" : "rename");
+                onEvent(path.resolve(dir, rel), eventType === "change" ? "change" : "rename");
             });
-            bootstrap.on("error", onError);
+            handle.on("error", onError);
+            bootstrap.push({ handle, dir, recursive });
         }
         catch (err) {
             // Recursive watch is unavailable at all — the plan below cannot work
@@ -1001,7 +1024,64 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             attachFailures += 1;
             onError(err);
         }
-    }
+    };
+    // Each directory is listed once even though the split asks about it from
+    // every ancestor level.
+    const bootstrapListings = new Map();
+    const listBootstrapChildren = (dir) => {
+        const cached = bootstrapListings.get(dir);
+        if (cached)
+            return cached;
+        let children;
+        try {
+            children = opts.listChildDirs
+                ? opts.listChildDirs(dir)
+                : fs
+                    .readdirSync(dir, { withFileTypes: true })
+                    .filter((entry) => entry.isDirectory())
+                    .map((entry) => path.join(dir, entry.name));
+        }
+        catch {
+            children = [];
+        }
+        bootstrapListings.set(dir, children);
+        return children;
+    };
+    // Only the heavy directories (worktrees, node_modules, nested .git) are split
+    // around. The emit filter is not consulted here: no part of the tree under
+    // the split is watched until the split finishes, so it has to stay as short
+    // as a directory listing. Filter-excluded directories are covered for the
+    // planning window and their events dropped by the filter, as before.
+    const bootstrapExcluded = (dir) => heavyWatchKind(path.relative(root, dir)) !== null;
+    /** True when an excluded directory sits within `levels` levels below `dir`. */
+    const hasExcludedWithin = (dir, levels) => {
+        if (levels <= 0)
+            return false;
+        for (const child of listBootstrapChildren(dir)) {
+            if (bootstrapExcluded(child) || hasExcludedWithin(child, levels - 1)) {
+                return true;
+            }
+        }
+        return false;
+    };
+    const coverForBootstrap = (dir, depth) => {
+        if (depth >= BOOTSTRAP_SPLIT_DEPTH) {
+            openBootstrap(dir, true);
+            return;
+        }
+        if (dir !== root && !hasExcludedWithin(dir, BOOTSTRAP_SPLIT_DEPTH - depth)) {
+            openBootstrap(dir, true);
+            return;
+        }
+        openBootstrap(dir, false);
+        for (const child of listBootstrapChildren(dir)) {
+            if (!bootstrapExcluded(child))
+                coverForBootstrap(child, depth + 1);
+        }
+    };
+    if (recursiveWatches)
+        coverForBootstrap(root, 0);
+    bootstrapListings.clear();
     if (!recursiveWatches) {
         addWatch(root, false);
         void planShallowAndAttach(root, true);
@@ -1020,29 +1100,20 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
         // them, reopening the very gap the bootstrap watch exists to close. Hold it
         // open for one drain window instead; the overlap only costs duplicate events
         // for excluded paths, which the emit filter drops and the debounce coalesces.
-        if (bootstrap) {
-            const handle = bootstrap;
-            bootstrap = null;
-            const timer = setTimeout(() => {
-                try {
-                    handle.close();
-                }
-                catch {
-                    /* already closed */
-                }
+        if (bootstrap.length > 0) {
+            const handles = bootstrap.map(({ handle }) => handle);
+            const closeAll = () => {
+                for (const handle of handles)
+                    closeHandle(handle);
+                bootstrap.length = 0;
                 closeBootstrap = () => { };
-            }, BOOTSTRAP_DRAIN_MS);
+            };
+            const timer = setTimeout(closeAll, BOOTSTRAP_DRAIN_MS);
             // Never hold the process open just to retire a temporary watch.
             timer.unref?.();
             closeBootstrap = () => {
                 clearTimeout(timer);
-                try {
-                    handle.close();
-                }
-                catch {
-                    /* already closed */
-                }
-                closeBootstrap = () => { };
+                closeAll();
             };
         }
     }
@@ -1081,6 +1152,13 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             return () => readyListeners.delete(listener);
         },
         watchedPathCount: () => watchers.size,
+        watchedDirectories: () => [
+            ...bootstrap.map(({ dir, recursive }) => ({ path: dir, recursive })),
+            ...[...watchers.keys()].map((dir) => ({
+                path: dir,
+                recursive: recursiveFlag.get(dir) === true,
+            })),
+        ],
         attachFailures,
         probeCoverage: () => probeScopedWatchCoverage({
             root,
@@ -1168,6 +1246,7 @@ export function startNativeRootWatch(hqRoot, shouldEmit, onEvent, onError, opts
         needsKnownKinds: true,
         attachFailures,
         watchedPathCount: () => (handle ? 1 : 0),
+        watchedDirectories: () => (handle ? [{ path: root, recursive: true }] : []),
         probeCoverage: () => probeScopedWatchCoverage({
             root,
             identities: identity,
@@ -1248,21 +1327,42 @@ export function startTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatchedP
     return startChokidarTreeWatch(root, watchFilter, watchEvent, onError, maxWatchedPaths, onWatchBudgetExceeded, forcePolling);
 }
 /**
- * Build the composite emit-decision predicate. Returns true when a change to
- * `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
- * layer). Pure and chokidar-free so the matching logic is unit-testable
- * directly.
+ * Classify a root-relative path that the watcher must never watch, at any
+ * depth: git worktree roots (`worktrees/` at the top level, `.worktrees/`,
+ * `workspace/worktrees/` and `.claude/worktrees/` anywhere), `node_modules/`,
+ * and a `.git` below the root. Returns the kind for the path itself or for an
+ * ancestor, or null when the path is not inside one.
  *
- * @param hqRoot       sync root (== personal-vault root in personalMode).
- * @param personalMode when true, also applies the personal-vault default
- *                     exclusions and the excluded-top-level buckets.
- * @param options.includeCompanyPaths retains `companies/<slug>/` below that
- *                     otherwise-excluded top-level bucket. The combined
- *                     `--companies` runner needs this: it syncs company
- *                     vaults and the personal vault in one process, while the
- *                     personal side must still prune unrelated `workspace/`
- *                     churn.
+ * A plain `worktrees` folder below the top level is not matched, so a
+ * knowledge folder with that name keeps live watching (the DEV-1791 rule the
+ * sync ignore list follows for the same reason).
  */
+export function heavyWatchKind(relativePath) {
+    // Compared lower-cased: Windows and macOS file systems are case-insensitive,
+    // so `Worktrees`, `.Worktrees` and `Node_Modules` are the same folders.
+    const segments = relativePath
+        .split(/[\\/]+/)
+        .filter((segment) => segment !== "")
+        .map((segment) => segment.toLowerCase());
+    if (segments[0] === "worktrees")
+        return "worktrees";
+    for (let i = 0; i < segments.length; i += 1) {
+        const segment = segments[i];
+        if (segment === "node_modules")
+            return "node_modules";
+        if (segment === ".worktrees")
+            return "worktrees";
+        if (segment === "worktrees" &&
+            (segments[i - 1] === "workspace" || segments[i - 1] === ".claude")) {
+            return "worktrees";
+        }
+        // The HQ root's own .git is excluded by the ignore stack; one below the
+        // root is a nested checkout.
+        if (segment === ".git" && i > 0)
+            return "git";
+    }
+    return null;
+}
 export function createWatchPathFilter(hqRoot, personalMode = false, options = {}) {
     const ignoreFilter = createIgnoreFilter(hqRoot);
     const personalScope = personalMode
@@ -1290,6 +1390,15 @@ export function createWatchPathFilter(hqRoot, personalMode = false, options = {}
         // The root itself, or anything outside it, is never an emit target.
         if (rel === "" || rel.startsWith(".."))
             return false;
+        // Worktrees, node_modules and nested .git are never watched, at any depth
+        // and in every mode. Checked before the ignore stack and every carve-out
+        // so no .hqinclude or workspace carve-out can re-admit them.
+        // A file probe checks only its folders: `node_modules/` is directory-only
+        // in the ignore stack, and chokidar's pre-stat probe is file-shaped for a
+        // new directory, so the stat-aware directory verdict decides that case.
+        const watchDirs = isDir ? rel : rel.slice(0, Math.max(0, rel.lastIndexOf("/")));
+        if (heavyWatchKind(watchDirs) !== null)
+            return false;
         // Layer 1: shared ignore stack (.hqignore/.gitignore/DEFAULT_IGNORES).
         // createIgnoreFilter returns true = "sync this"; false = "ignored".
         if (!ignoreFilter(absolutePath, isDir))
@@ -1511,6 +1620,9 @@ export class TreeWatcher {
     smallBatchMaxPaths;
     smallBatchMaxBytes;
     maxWaitMs;
+    minEmitGapMs;
+    /** clock.now() of the last batch handed to listeners; null before the first. */
+    lastEmitAt = null;
     clock;
     shouldEmit;
     maxPendingPaths;
@@ -1591,6 +1703,10 @@ export class TreeWatcher {
             opts.maxWaitMs !== undefined && opts.maxWaitMs > 0
                 ? Math.max(opts.maxWaitMs, this.debounceMs)
                 : Infinity;
+        this.minEmitGapMs =
+            opts.minEmitGapMs !== undefined && Number.isFinite(opts.minEmitGapMs) && opts.minEmitGapMs > 0
+                ? Math.floor(opts.minEmitGapMs)
+                : 0;
         this.clock = opts.clock ?? systemClock;
         this.shouldEmit =
             opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false, {
@@ -2322,9 +2438,15 @@ export class TreeWatcher {
             : this.debounceMs;
         // Quiet-period debounce, capped by the window's max-wait deadline so a
         // continuous event stream cannot postpone the emit forever.
-        const delayMs = this.windowDeadlineAt === null
+        const settleMs = this.windowDeadlineAt === null
             ? quietWindowMs
             : Math.max(0, Math.min(quietWindowMs, this.windowDeadlineAt - now));
+        // The pass-gap cap wins over both: a batch never emits sooner than
+        // minEmitGapMs after the previous one.
+        const gapMs = this.lastEmitAt === null || this.minEmitGapMs === 0
+            ? 0
+            : Math.max(0, this.lastEmitAt + this.minEmitGapMs - now);
+        const delayMs = Math.max(settleMs, gapMs);
         this.timer = this.clock.setTimeout(() => {
             this.timer = null;
             this.emit();
@@ -2360,6 +2482,7 @@ export class TreeWatcher {
         // undefined when the window settled with no captured path (e.g. a synthetic
         // arm() with no handleEvent).
         const firstRel = [...batch.paths.values()][0];
+        this.lastEmitAt = this.clock.now();
         for (const l of this.listeners) {
             try {
                 l(firstRel, batch);
@@ -2757,6 +2880,16 @@ export class TreeWatcher {
     watchedPathCount() {
         return this.backend?.watchedPathCount() ?? 0;
     }
+    /** OS-level watches held by the active backend, for diagnostics. */
+    watchedDirectories() {
+        return this.backend?.watchedDirectories?.() ?? [];
+    }
+    /** Backend mode before folding native-root into native. */
+    watchBackend() {
+        if (!this.backend)
+            return "inactive";
+        return this.backend.mode ?? "unknown";
+    }
     /** Active backend mode, including the no-inotify polling fallback. */
     watchMode() {
         const mode = this.backend?.mode;
@@ -2869,6 +3002,25 @@ export const DEFAULT_PUBLISHED_CONTENT_HASH_MAX_PATHS = 50_000;
 /** Minimum delay between successful upsert announcements for one path. */
 export const DEFAULT_PUBLISH_MIN_INTERVAL_MS = 60_000;
 export const PUBLISH_MIN_INTERVAL_MS_ENV = "HQ_SYNC_PUBLISH_MIN_INTERVAL_MS";
+/**
+ * Minimum time between two watcher batches, and so between two
+ * watch-triggered passes. Events that arrive inside the gap are held and
+ * merged into the next batch, which emits once the gap has passed. This caps
+ * the pass rate even when the debounce windows are tuned short or a stream of
+ * small batches keeps settling. `HQ_SYNC_MIN_PASS_GAP_MS` overrides it; `0`
+ * turns the cap off, and an invalid value falls back to the default.
+ */
+export const DEFAULT_MIN_PASS_GAP_MS = 30_000;
+export const MIN_PASS_GAP_MS_ENV = "HQ_SYNC_MIN_PASS_GAP_MS";
+export function resolveMinPassGapMs(env = process.env) {
+    const raw = env[MIN_PASS_GAP_MS_ENV];
+    if (raw === undefined || raw.trim() === "")
+        return DEFAULT_MIN_PASS_GAP_MS;
+    const value = Number(raw);
+    return Number.isFinite(value) && value >= 0
+        ? Math.floor(value)
+        : DEFAULT_MIN_PASS_GAP_MS;
+}
 /** Resolve the optional per-path announcement interval without accepting invalid values. */
 export function resolvePublishMinIntervalMs(env = process.env) {
     const raw = env[PUBLISH_MIN_INTERVAL_MS_ENV];