@indigoai-us/hq-cloud 6.18.33 → 6.18.35

package/dist/watcher.js

@@ -528,6 +528,16 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
     let budgetExhausted = false;
     /** True once the initial plan is live; later attaches announce their files. */
     let planned = false;
+    let initialDiscoveryComplete = false;
+    const readyListeners = new Set();
+    const notifyReady = () => {
+        if (initialDiscoveryComplete || closed)
+            return;
+        initialDiscoveryComplete = true;
+        for (const listener of readyListeners)
+            listener();
+        readyListeners.clear();
+    };
     /** Retires the bootstrap watch — replaced once that watch exists. */
     let closeBootstrap = () => { };
     /**
@@ -586,6 +596,8 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
     }
     /** `dev:ino` of the directory at `absolutePath`; undefined if absent or not a directory. */
     function directoryIdentity(absolutePath) {
+        if (opts.directoryIdentity)
+            return opts.directoryIdentity(absolutePath);
         try {
             const st = fs.lstatSync(absolutePath);
             return st.isDirectory() ? `${st.dev}:${st.ino}` : undefined;
@@ -719,7 +731,7 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             addWatch(dir, false);
     }
     /**
-     * Plan `dir` and attach the result. In shallow mode every in-scope
+     * Plan `dir` and attach the result on recursive-watch platforms. In shallow mode every in-scope
      * directory is attached the moment the walk discovers it — BEFORE the walk
      * lists that directory — rather than after the whole plan is computed. The
      * plan is a snapshot: a child created after its parent was listed is in
@@ -747,6 +759,66 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             listChildDirs: opts.listChildDirs,
         }));
     }
+    /**
+     * Linux needs one inotify watch per included directory. Keep the parent
+     * attached before listing it, but yield regularly so a large root cannot
+     * hold the runner's event loop while tens of thousands of watches open.
+     */
+    async function planShallowAndAttach(dir, initial = false) {
+        const pending = [dir];
+        let workSinceYield = 0;
+        const yieldIfNeeded = async () => {
+            workSinceYield += 1;
+            if (workSinceYield < 128)
+                return closed;
+            workSinceYield = 0;
+            await new Promise((resolve) => setImmediate(resolve));
+            return closed;
+        };
+        while (pending.length > 0 && !closed) {
+            const current = pending.pop();
+            try {
+                if (opts.listChildDirs) {
+                    for (const child of opts.listChildDirs(current)) {
+                        if (shouldEmit(child, true)) {
+                            onDirectorySeen?.(path.resolve(child));
+                            addWatch(child, false);
+                            pending.push(child);
+                        }
+                        if (await yieldIfNeeded())
+                            return;
+                    }
+                }
+                else {
+                    const entries = await fs.promises.readdir(current, { withFileTypes: true });
+                    for (const entry of entries) {
+                        const absolutePath = path.join(current, entry.name);
+                        if (entry.isDirectory()) {
+                            if (shouldEmit(absolutePath, true)) {
+                                onDirectorySeen?.(path.resolve(absolutePath));
+                                addWatch(absolutePath, false);
+                                pending.push(absolutePath);
+                            }
+                        }
+                        else if (!initial && shouldEmit(absolutePath, false)) {
+                            onEvent(absolutePath, "rename");
+                        }
+                        if (await yieldIfNeeded())
+                            return;
+                    }
+                }
+            }
+            catch (err) {
+                const code = err?.code;
+                if (code !== "ENOENT" && code !== "ENOTDIR")
+                    onError(err);
+            }
+        }
+        if (initial) {
+            planned = true;
+            notifyReady();
+        }
+    }
     /**
      * Announce every in-scope file already inside a directory that was just
      * attached. A shallow parent watch reports only that the directory
@@ -853,6 +925,11 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             return;
         const attachedAt = Date.now();
         onDirectorySeen?.(absolutePath);
+        if (!recursiveWatches) {
+            addWatch(absolutePath, false);
+            void planShallowAndAttach(absolutePath);
+            return;
+        }
         planAndAttach(absolutePath);
         if (!planned)
             return;
@@ -903,7 +980,7 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
     // dropped the moment the scoped plan is live.
     // No such watch exists in shallow mode: Node's Linux "recursive" shim would
     // walk the WHOLE tree — worktrees, node_modules and all — to emulate it.
-    // The runner's initial sync pass covers the planning window there instead.
+    // In shallow mode, each parent is attached before its child list is read.
     let bootstrap = null;
     if (recursiveWatches) {
         try {
@@ -924,42 +1001,49 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
             onError(err);
         }
     }
-    // Unbounded on purpose: this walk replaces TreeWatcher's separate known-kinds
-    // seed walk (same tree, same filter, ~3.6s on a real HQ root), so planning
-    // the tree exhaustively costs nothing over the status quo and leaves no
-    // excluded subtree covered by a recursive root.
-    planAndAttach(root);
-    planned = true;
-    // The scoped plan is live, so narrow from whole-tree to in-scope-only — but
-    // NOT synchronously. Events observed during the planning walk can still be
-    // sitting in the OS queue undelivered, and closing the handle here drops
-    // them, reopening the very gap the bootstrap watch exists to close. Hold it
-    // open for one drain window instead; the overlap only costs duplicate events
-    // for excluded paths, which the emit filter drops and the debounce coalesces.
-    if (bootstrap) {
-        const handle = bootstrap;
-        bootstrap = null;
-        const timer = setTimeout(() => {
-            try {
-                handle.close();
-            }
-            catch {
-                /* already closed */
-            }
-            closeBootstrap = () => { };
-        }, BOOTSTRAP_DRAIN_MS);
-        // Never hold the process open just to retire a temporary watch.
-        timer.unref?.();
-        closeBootstrap = () => {
-            clearTimeout(timer);
-            try {
-                handle.close();
-            }
-            catch {
-                /* already closed */
-            }
-            closeBootstrap = () => { };
-        };
+    if (!recursiveWatches) {
+        addWatch(root, false);
+        void planShallowAndAttach(root, true);
+    }
+    else {
+        // Unbounded on purpose: this walk replaces TreeWatcher's separate known-kinds
+        // seed walk (same tree, same filter, ~3.6s on a real HQ root), so planning
+        // the tree exhaustively costs nothing over the status quo and leaves no
+        // excluded subtree covered by a recursive root.
+        planAndAttach(root);
+        planned = true;
+        notifyReady();
+        // The scoped plan is live, so narrow from whole-tree to in-scope-only — but
+        // NOT synchronously. Events observed during the planning walk can still be
+        // sitting in the OS queue undelivered, and closing the handle here drops
+        // them, reopening the very gap the bootstrap watch exists to close. Hold it
+        // open for one drain window instead; the overlap only costs duplicate events
+        // for excluded paths, which the emit filter drops and the debounce coalesces.
+        if (bootstrap) {
+            const handle = bootstrap;
+            bootstrap = null;
+            const timer = setTimeout(() => {
+                try {
+                    handle.close();
+                }
+                catch {
+                    /* already closed */
+                }
+                closeBootstrap = () => { };
+            }, BOOTSTRAP_DRAIN_MS);
+            // Never hold the process open just to retire a temporary watch.
+            timer.unref?.();
+            closeBootstrap = () => {
+                clearTimeout(timer);
+                try {
+                    handle.close();
+                }
+                catch {
+                    /* already closed */
+                }
+                closeBootstrap = () => { };
+            };
+        }
     }
     return {
         close: () => {
@@ -987,6 +1071,14 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
         mode: "native",
         needsKnownKinds: true,
         seededKnownKinds: onDirectorySeen !== undefined,
+        onReady: (listener) => {
+            if (initialDiscoveryComplete) {
+                listener();
+                return () => { };
+            }
+            readyListeners.add(listener);
+            return () => readyListeners.delete(listener);
+        },
         watchedPathCount: () => watchers.size,
         attachFailures,
         probeCoverage: () => probeScopedWatchCoverage({
@@ -1945,7 +2037,7 @@ export class TreeWatcher {
         if (marker)
             marker.epoch += 1;
     }
-    enqueueDeleteSnapshotCapture(absolutePath, relativePath, kind) {
+    enqueueDeleteSnapshotCapture(absolutePath, relativePath, kind, deferBatchSchedule = false) {
         // This delete supersedes whatever capture is already in flight for the
         // path, whether or not the queue has room for a new one.
         const live = this.deleteSnapshotEpochs.get(absolutePath);
@@ -1967,7 +2059,7 @@ export class TreeWatcher {
         this.deleteSnapshotEpochs.set(absolutePath, marker);
         const isCurrent = () => this.deleteSnapshotEpochs.get(absolutePath)?.epoch === epoch;
         if (this.queueLocalDeleteSnapshotsBatch) {
-            return this.enqueueDeleteSnapshotCaptureBatch(absolutePath, { relativePath, kind, isCurrent });
+            return this.enqueueDeleteSnapshotCaptureBatch(absolutePath, { relativePath, kind, isCurrent }, !deferBatchSchedule);
         }
         const run = this.deleteSnapshotCaptureTail.then(() => this.queueLocalDeleteSnapshots(relativePath, kind, isCurrent), () => this.queueLocalDeleteSnapshots(relativePath, kind, isCurrent));
         this.deleteSnapshotCaptureTail = run.then(() => undefined, () => undefined);
@@ -1975,7 +2067,7 @@ export class TreeWatcher {
         void capture.catch(() => undefined);
         return capture;
     }
-    enqueueDeleteSnapshotCaptureBatch(absolutePath, capture) {
+    enqueueDeleteSnapshotCaptureBatch(absolutePath, capture, schedule = true) {
         const promise = new Promise((resolve, reject) => {
             this.pendingDeleteSnapshotCaptureBatch.push({
                 absolutePath,
@@ -1985,7 +2077,8 @@ export class TreeWatcher {
             });
         });
         void promise.catch(() => undefined);
-        this.scheduleDeleteSnapshotCaptureBatch();
+        if (schedule)
+            this.scheduleDeleteSnapshotCaptureBatch();
         return promise;
     }
     scheduleDeleteSnapshotCaptureBatch() {
@@ -2345,22 +2438,29 @@ export class TreeWatcher {
             cancelled: true,
         });
         const yieldToEventLoop = async () => {
-            if (cancelled())
+            if (cancelled()) {
+                if (this.pendingDeleteSnapshotCaptureBatch.length > 0) {
+                    this.scheduleDeleteSnapshotCaptureBatch();
+                }
                 return false;
+            }
             workSinceYield += 1;
             if (workSinceYield < yieldEvery)
                 return true;
             workSinceYield = 0;
+            if (this.pendingDeleteSnapshotCaptureBatch.length > 0) {
+                this.scheduleDeleteSnapshotCaptureBatch();
+            }
             await new Promise((resolve) => setImmediate(resolve));
             return !cancelled();
         };
-        const record = (absolutePath, kind) => {
+        const record = async (absolutePath, kind) => {
             const absolute = path.resolve(absolutePath);
             const relative = vaultKeyForLocalPath(this.hqRoot, absolute);
             observed.add(relative);
             let entry;
             try {
-                entry = fs.lstatSync(absolute);
+                entry = await fs.promises.lstat(absolute);
             }
             catch {
                 return;
@@ -2376,7 +2476,7 @@ export class TreeWatcher {
                 return false;
             let entries;
             try {
-                entries = fs.readdirSync(dir, { withFileTypes: true });
+                entries = await fs.promises.readdir(dir, { withFileTypes: true });
             }
             catch {
                 return !cancelled();
@@ -2396,7 +2496,7 @@ export class TreeWatcher {
                 if (entry.isFile() || entry.isSymbolicLink()) {
                     // A symlink may be admitted only by a directory-only include rule.
                     if (this.shouldEmit(absolute, false) || this.shouldEmit(absolute, true)) {
-                        record(absolute, "change");
+                        await record(absolute, "change");
                     }
                 }
                 if (!(await yieldToEventLoop()))
@@ -2421,8 +2521,17 @@ export class TreeWatcher {
                     return cancelledResult();
                 continue;
             }
-            if (isPresentOnDisk(absolute)) {
-                record(absolute, "change");
+            let present = false;
+            try {
+                await fs.promises.lstat(absolute);
+                present = true;
+            }
+            catch {
+                // Missing paths become deletes; other errors preserve the previous
+                // behavior of treating an unreadable path as absent.
+            }
+            if (present) {
+                await record(absolute, "change");
                 if (!(await yieldToEventLoop()))
                     return cancelledResult();
                 continue;
@@ -2432,7 +2541,7 @@ export class TreeWatcher {
             let deleteSnapshotCapture;
             try {
                 if (this.queueLocalDeleteSnapshotsBatch || this.queueLocalDeleteSnapshots) {
-                    deleteSnapshotCapture = this.enqueueDeleteSnapshotCapture(absolute, relative, "unlink");
+                    deleteSnapshotCapture = this.enqueueDeleteSnapshotCapture(absolute, relative, "unlink", true);
                 }
                 else {
                     deleteSnapshots = this.captureLocalDeleteSnapshots?.(relative, "unlink") ?? [];
@@ -2449,6 +2558,13 @@ export class TreeWatcher {
             if (!(await yieldToEventLoop()))
                 return cancelledResult();
         }
+        // Keep the warm-up rows from the same bounded slice together before the
+        // queued snapshot callback's setImmediate flush runs. Promise-based lstat
+        // otherwise spaces successive rows across I/O turns and splits one logical
+        // warm-up batch into a sequence of one-row captures.
+        if (this.pendingDeleteSnapshotCaptureBatch.length > 0) {
+            this.scheduleDeleteSnapshotCaptureBatch();
+        }
         return { batch: { paths, changes }, scannedPaths };
     }
     /** True while the watch backend is active. */