@indigoai-us/hq-cloud 6.11.11 → 6.11.13

package/src/watcher.test.ts

@@ -49,6 +49,10 @@ function makeHarness(opts?: { debounceMs?: number }) {
   };
 }
 
+async function flushImmediate(): Promise<void> {
+  await new Promise<void>((resolve) => setImmediate(resolve));
+}
+
 describe("FakeClock", () => {
   it("fires a timer exactly when its deadline is reached", () => {
     const clock = new FakeClock();
@@ -166,6 +170,34 @@ describe("US-001: WatchPushDriver — debounced push seam", () => {
     driver.dispose();
   });
 
+  it("F17: rejected watch push is caught instead of surfacing as an unhandled rejection", async () => {
+    const clock = new FakeClock();
+    const rejection = new Error("push failed");
+    const push = vi.fn(async () => {
+      throw rejection;
+    });
+    const driver = new WatchPushDriver({ debounceMs: DEBOUNCE, clock, push });
+    const unhandled: unknown[] = [];
+    const onUnhandled = (reason: unknown) => {
+      unhandled.push(reason);
+    };
+
+    process.prependListener("unhandledRejection", onUnhandled);
+    try {
+      driver.notifyChange();
+      clock.advance(DEBOUNCE);
+      await Promise.resolve();
+      await flushImmediate();
+
+      expect(push).toHaveBeenCalledTimes(1);
+      expect(driver.isPushing()).toBe(false);
+      expect(unhandled).toEqual([]);
+    } finally {
+      process.removeListener("unhandledRejection", onUnhandled);
+      driver.dispose();
+    }
+  });
+
   it("respects a custom debounce window", async () => {
     const h = makeHarness({ debounceMs: 500 });
     h.emitChange();
@@ -374,6 +406,42 @@ describe("US-002: TreeWatcher — debounce coalesce (FakeClock seam)", () => {
     clock.advance(DEBOUNCE);
     expect(changed).toHaveBeenCalledTimes(2);
   });
+
+  it("R-F13: caps watcher backlog and keeps EMFILE polling fallback removed", () => {
+    const clock = new FakeClock();
+    const changed = vi.fn();
+    const overflow = vi.fn();
+    const watcher = new TreeWatcher({
+      hqRoot: ROOT,
+      debounceMs: DEBOUNCE,
+      clock,
+      pathFilter: () => true,
+      maxPendingPaths: 3,
+      maxPendingBytes: 10_000,
+      onBacklogOverflow: overflow,
+    });
+    watcher.onChange(changed);
+
+    for (let i = 0; i < 10; i++) {
+      watcher.handleEvent(path.join(ROOT, `bulk-${i}.md`));
+    }
+
+    expect(overflow).toHaveBeenCalledTimes(1);
+    clock.advance(DEBOUNCE);
+    expect(changed).toHaveBeenCalledTimes(1);
+    const batch = changed.mock.calls[0][1] as {
+      paths: Map<string, string>;
+      overflowed?: boolean;
+      droppedPaths?: number;
+    };
+    expect(batch.paths.size).toBe(3);
+    expect(batch.overflowed).toBe(true);
+    expect(batch.droppedPaths).toBe(7);
+
+    const source = fs.readFileSync(path.join(process.cwd(), "src/watcher.ts"), "utf8");
+    expect(source).not.toContain("startPollingTreeWatch");
+    expect(source).not.toContain("snapshotWatchTree");
+  });
 });
 
 describe("US-002: TreeWatcher — lifecycle (real chokidar over a temp dir)", () => {
@@ -468,6 +536,55 @@ describe("PushEventEmitter — directory and delete tombstone handling", () => {
     });
   }
 
+  it("F13: watcher batch event publishing applies bounded backpressure", async () => {
+    const paths = new Map<string, string>();
+    for (let i = 0; i < 32; i++) {
+      const rel = `bulk-${i}.md`;
+      const abs = path.join(dir, rel);
+      fs.writeFileSync(abs, `file ${i}`);
+      paths.set(abs, rel);
+    }
+
+    let inFlight = 0;
+    let maxInFlight = 0;
+    let releaseGate!: () => void;
+    const gate = new Promise<void>((resolve) => {
+      releaseGate = resolve;
+    });
+    const transport: PushTransport = {
+      start: async () => {},
+      dispose: async () => {},
+      connected: true,
+      publish: async () => {
+        inFlight += 1;
+        maxInFlight = Math.max(maxInFlight, inFlight);
+        await gate;
+        inFlight -= 1;
+      },
+    };
+    const emitter = new PushEventEmitter({
+      originTenantId: "tenant-indigo",
+      originDeviceId: "device-a",
+      transport,
+      flagProvider: new StaticFlagProvider(["tenant-indigo"]),
+      now: () => new Date("2026-06-18T12:00:00.000Z"),
+    });
+
+    const run = emitter.emitForBatch({ paths });
+    try {
+      for (let i = 0; i < 200 && maxInFlight <= 16; i++) {
+        await flushImmediate();
+      }
+      releaseGate();
+      await run;
+
+      expect(maxInFlight).toBeLessThanOrEqual(16);
+    } finally {
+      releaseGate();
+      await run.catch(() => {});
+    }
+  });
+
   it("skips directory changes silently without publishing or reporting an error", async () => {
     const published: PushEvent[] = [];
     const onError = vi.fn();

package/src/watcher.ts

@@ -2,9 +2,8 @@
  * File watcher — monitors HQ directory for changes
  * Uses chokidar with debounced batching
  *
- * Day 1: not invoked by CLI surface; retained for future automatic-sync milestone.
- * When re-enabled, the constructor will need an EntityContext (or a context resolver)
- * to be passed in for entity-aware S3 operations.
+ * Active watcher path: TreeWatcher detects local changes, WatchPushDriver
+ * schedules pushes, and PushEventEmitter publishes typed push events.
  */
 
 import * as fs from "fs";
@@ -12,12 +11,7 @@ import { createHash } from "node:crypto";
 import { readFile, stat } from "node:fs/promises";
 import * as path from "path";
 import { watch } from "chokidar";
-import type { FSWatcher } from "chokidar";
-import type { EntityContext } from "./types.js";
-import { createIgnoreFilter, isWithinSizeLimit } from "./ignore.js";
-import { readJournal, writeJournal, hashFile, updateEntry } from "./journal.js";
-import { uploadFile, deleteRemoteFile, toPosixKey } from "./s3.js";
-import type { UploadAuthor } from "./s3.js";
+import { createIgnoreFilter } from "./ignore.js";
 import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
 import {
   CONTINUITY_POINTER_REL,
@@ -163,7 +157,9 @@ export class WatchPushDriver {
     }
     this.timer = this.clock.setTimeout(() => {
       this.timer = null;
-      void this.fire();
+      void this.fire().catch((err) => {
+        console.error("WatchPushDriver push failed:", err);
+      });
     }, this.debounceMs);
   }
 
@@ -203,141 +199,12 @@ export class WatchPushDriver {
   }
 }
 
-interface PendingChange {
-  type: "add" | "change" | "unlink";
-  absolutePath: string;
-  relativePath: string;
-}
-
-export class SyncWatcher {
-  private watcher: FSWatcher | null = null;
-  private hqRoot: string;
-  private ctx: EntityContext;
-  private author?: UploadAuthor;
-  private shouldSync: (filePath: string, isDir?: boolean) => boolean;
-  private pendingChanges = new Map<string, PendingChange>();
-  private debounceTimer: ReturnType<typeof setTimeout> | null = null;
-  private processing = false;
-
-  constructor(hqRoot: string, ctx: EntityContext, author?: UploadAuthor) {
-    this.hqRoot = hqRoot;
-    this.ctx = ctx;
-    this.author = author;
-    this.shouldSync = createIgnoreFilter(hqRoot);
-  }
-
-  start(): void {
-    if (this.watcher) return;
-
-    this.watcher = watch(this.hqRoot, {
-      // See toChokidarIgnored: chokidar's pre-stat descent probe has no stats
-      // hint, so a naive file-verdict prunes intermediate allowlist dirs
-      // before descending to their in-scope leaves. Keep a statless probe
-      // when EITHER the file or directory reading would survive the filter.
-      ignored: toChokidarIgnored(this.shouldSync, this.hqRoot),
-      persistent: true,
-      ignoreInitial: true,
-      awaitWriteFinish: {
-        stabilityThreshold: 500,
-        pollInterval: 100,
-      },
-    });
-
-    this.watcher
-      .on("add", (p) => this.queueChange("add", p))
-      .on("change", (p) => this.queueChange("change", p))
-      .on("unlink", (p) => this.queueChange("unlink", p))
-      .on("error", (err) => console.error("Watcher error:", err));
-  }
-
-  stop(): void {
-    if (this.watcher) {
-      this.watcher.close();
-      this.watcher = null;
-    }
-    if (this.debounceTimer) {
-      clearTimeout(this.debounceTimer);
-      this.debounceTimer = null;
-    }
-  }
-
-  private queueChange(type: "add" | "change" | "unlink", absolutePath: string): void {
-    const relativePath = toPosixKey(path.relative(this.hqRoot, absolutePath));
-
-    // Skip files that exceed size limit
-    if (type !== "unlink" && !isWithinSizeLimit(absolutePath)) {
-      return;
-    }
-
-    this.pendingChanges.set(relativePath, {
-      type,
-      absolutePath,
-      relativePath,
-    });
-
-    // Debounce: wait for DEBOUNCE_MS of quiet before processing
-    if (this.debounceTimer) {
-      clearTimeout(this.debounceTimer);
-    }
-    this.debounceTimer = setTimeout(() => this.flush(), DEBOUNCE_MS);
-  }
-
-  private async flush(): Promise<void> {
-    if (this.processing || this.pendingChanges.size === 0) return;
-    this.processing = true;
-
-    const batch = new Map(this.pendingChanges);
-    this.pendingChanges.clear();
-
-    const journal = readJournal(this.ctx.slug);
-
-    for (const [relativePath, change] of batch) {
-      try {
-        if (change.type === "unlink") {
-          await deleteRemoteFile(this.ctx, relativePath);
-          delete journal.files[relativePath];
-        } else {
-          const hash = hashFile(change.absolutePath);
-          const stat = fs.statSync(change.absolutePath);
-
-          // Skip if unchanged from last sync
-          const existing = journal.files[relativePath];
-          if (existing && existing.hash === hash) continue;
-
-          const { etag } = this.author
-            ? await uploadFile(this.ctx, change.absolutePath, relativePath, this.author)
-            : await uploadFile(this.ctx, change.absolutePath, relativePath);
-          updateEntry(journal, relativePath, hash, stat.size, "up", etag);
-        }
-      } catch (err) {
-        console.error(
-          `Sync error [${relativePath}]:`,
-          err instanceof Error ? err.message : err
-        );
-        // Re-queue failed changes
-        this.pendingChanges.set(relativePath, change);
-      }
-    }
-
-    // See cli/sync.ts: stamp lastSync on every flush so the indicator
-    // ticks even when all changes were re-queued or no-op.
-    journal.lastSync = new Date().toISOString();
-    writeJournal(this.ctx.slug, journal);
-    this.processing = false;
-
-    // Process any changes that came in while we were flushing
-    if (this.pendingChanges.size > 0) {
-      this.debounceTimer = setTimeout(() => this.flush(), DEBOUNCE_MS);
-    }
-  }
-}
-
 // ---------------------------------------------------------------------------
 // US-002 — debounced, ignore-aware, exclusion-aware tree watcher.
 //
-// Unlike SyncWatcher (which does S3 work inline), TreeWatcher is a pure change
-// detector: it emits a single debounced `changed` callback after a quiet window
-// and never touches S3 itself. US-003 wires that callback to a targeted push.
+// TreeWatcher is a pure change detector: it emits a single debounced `changed`
+// callback after a quiet window and never touches S3 itself. US-003 wires that
+// callback to a targeted push.
 //
 // The emit decision composes the SAME filter stack the push walk uses, so a
 // path that the push would skip never wakes the watcher:
@@ -400,6 +267,36 @@ function supportsRecursiveWatch(): boolean {
   return process.platform === "darwin" || process.platform === "win32";
 }
 
+function startChokidarTreeWatch(
+  hqRoot: string,
+  shouldEmit: WatchPathFilter,
+  onEvent: (absolutePath: string) => void,
+  onError: (err: unknown) => void,
+): WatchBackend {
+  const cw = watch(hqRoot, {
+    // chokidar fallback (Linux): see toChokidarIgnored for why the descent
+    // probe must not prune ancestor dirs of allowlisted leaves.
+    ignored: toChokidarIgnored(shouldEmit, hqRoot),
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 500,
+      pollInterval: 100,
+    },
+  });
+  cw.on("add", onEvent)
+    .on("change", onEvent)
+    .on("unlink", onEvent)
+    .on("addDir", onEvent)
+    .on("unlinkDir", onEvent)
+    .on("error", onError);
+  return {
+    close: () => {
+      void cw.close();
+    },
+  };
+}
+
 /**
  * Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
  *
@@ -441,8 +338,39 @@ function startTreeWatch(
           onEvent(path.resolve(hqRoot, rel));
         },
       );
-      native.on("error", onError);
-      return { close: () => native.close() };
+      let closed = false;
+      let fallback: WatchBackend | null = null;
+      native.on("error", (err) => {
+        onError(err);
+        if (closed || fallback !== null) return;
+        try {
+          native.close();
+        } catch {
+          /* already closed */
+        }
+        try {
+          fallback = startChokidarTreeWatch(
+            hqRoot,
+            shouldEmit,
+            onEvent,
+            onError,
+          );
+        } catch (fallbackErr) {
+          onError(fallbackErr);
+        }
+      });
+      return {
+        close: () => {
+          closed = true;
+          try {
+            native.close();
+          } catch {
+            /* already closed */
+          }
+          fallback?.close();
+          fallback = null;
+        },
+      };
     } catch (err) {
       // Recursive watch unexpectedly unavailable — fall back to chokidar
       // rather than leaving the daemon with no watcher at all.
@@ -450,28 +378,7 @@ function startTreeWatch(
     }
   }
 
-  const cw = watch(hqRoot, {
-    // chokidar fallback (Linux): see toChokidarIgnored for why the descent
-    // probe must not prune ancestor dirs of allowlisted leaves.
-    ignored: toChokidarIgnored(shouldEmit, hqRoot),
-    persistent: true,
-    ignoreInitial: true,
-    awaitWriteFinish: {
-      stabilityThreshold: 500,
-      pollInterval: 100,
-    },
-  });
-  cw.on("add", onEvent)
-    .on("change", onEvent)
-    .on("unlink", onEvent)
-    .on("addDir", onEvent)
-    .on("unlinkDir", onEvent)
-    .on("error", onError);
-  return {
-    close: () => {
-      void cw.close();
-    },
-  };
+  return startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError);
 }
 
 /**
@@ -533,6 +440,25 @@ export function createWatchPathFilter(
   };
 }
 
+export const DEFAULT_TREE_WATCHER_MAX_PENDING_PATHS = 4096;
+export const DEFAULT_TREE_WATCHER_MAX_PENDING_BYTES = 1024 * 1024;
+
+export interface TreeWatcherBacklogOverflow {
+  pendingPaths: number;
+  pendingBytes: number;
+  maxPendingPaths: number;
+  maxPendingBytes: number;
+  droppedPaths: number;
+  droppedBytes: number;
+}
+
+function estimatePendingEntryBytes(
+  absolutePath: string,
+  relativePath: string,
+): number {
+  return Buffer.byteLength(absolutePath) + Buffer.byteLength(relativePath) + 64;
+}
+
 export interface TreeWatcherOptions {
   /** Sync root to watch (== personal-vault root in personalMode). */
   hqRoot: string;
@@ -547,6 +473,12 @@ export interface TreeWatcherOptions {
    * from {@link createWatchPathFilter}.
    */
   pathFilter?: WatchPathFilter;
+  /** Maximum distinct paths retained in one debounce window. */
+  maxPendingPaths?: number;
+  /** Approximate maximum path-string bytes retained in one debounce window. */
+  maxPendingBytes?: number;
+  /** Backlog overflow signal. Defaults to a console warning. */
+  onBacklogOverflow?: (info: TreeWatcherBacklogOverflow) => void;
 }
 
 /**
@@ -569,6 +501,12 @@ export interface TreeWatcherOptions {
 export interface TreeChangeBatch {
   /** Map of absolutePath → relativePath for every path in the settled burst. */
   paths: Map<string, string>;
+  /** True when path detail was dropped after the watcher backlog cap was hit. */
+  overflowed?: boolean;
+  /** Count of paths dropped from the detailed batch after overflow. */
+  droppedPaths?: number;
+  /** Approximate path-string bytes dropped from the detailed batch. */
+  droppedBytes?: number;
 }
 
 /**
@@ -591,11 +529,19 @@ export class TreeWatcher {
   private readonly debounceMs: number;
   private readonly clock: Clock;
   private readonly shouldEmit: WatchPathFilter;
+  private readonly maxPendingPaths: number;
+  private readonly maxPendingBytes: number;
+  private readonly onBacklogOverflow: (info: TreeWatcherBacklogOverflow) => void;
   private backend: WatchBackend | null = null;
   private timer: unknown = null;
   private listeners = new Set<TreeChangeListener>();
   /** Paths accumulated for the current (in-flight) debounce window. */
   private pending = new Map<string, string>();
+  private pendingBytes = 0;
+  private overflowed = false;
+  private overflowLogged = false;
+  private droppedPaths = 0;
+  private droppedBytes = 0;
   private disposed = false;
 
   constructor(opts: TreeWatcherOptions) {
@@ -604,6 +550,21 @@ export class TreeWatcher {
     this.clock = opts.clock ?? systemClock;
     this.shouldEmit =
       opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
+    this.maxPendingPaths = Math.max(
+      1,
+      opts.maxPendingPaths ?? DEFAULT_TREE_WATCHER_MAX_PENDING_PATHS,
+    );
+    this.maxPendingBytes = Math.max(
+      1,
+      opts.maxPendingBytes ?? DEFAULT_TREE_WATCHER_MAX_PENDING_BYTES,
+    );
+    this.onBacklogOverflow =
+      opts.onBacklogOverflow ??
+      ((info) => {
+        console.warn(
+          `TreeWatcher backlog cap exceeded; dropping ${info.droppedPaths} path(s) until the next resync`,
+        );
+      });
   }
 
   /**
@@ -633,10 +594,19 @@ export class TreeWatcher {
       this.hqRoot,
       this.shouldEmit,
       (absolutePath) => this.handleEvent(absolutePath),
-      (err) => console.error("TreeWatcher error:", err),
+      (err) => {
+        console.error("TreeWatcher error:", err);
+        this.signalBackendResync();
+      },
     );
   }
 
+  private signalBackendResync(): void {
+    if (this.disposed) return;
+    this.overflowed = true;
+    this.arm();
+  }
+
   /**
    * Test/seam entry point: feed a raw filesystem path as if the backend
    * reported it. Applies the emit filter then arms the debounce. Real watch
@@ -651,10 +621,42 @@ export class TreeWatcher {
     if (!this.shouldEmit(absolutePath, false)) return;
     const abs = path.resolve(absolutePath);
     const rel = path.relative(this.hqRoot, abs).split(path.sep).join("/");
+    if (!this.pending.has(abs)) {
+      const entryBytes = estimatePendingEntryBytes(abs, rel);
+      if (
+        this.pending.size >= this.maxPendingPaths ||
+        this.pendingBytes + entryBytes > this.maxPendingBytes
+      ) {
+        this.recordBacklogOverflow(entryBytes);
+        this.arm();
+        return;
+      }
+      this.pendingBytes += entryBytes;
+    }
     this.pending.set(abs, rel);
     this.arm();
   }
 
+  private recordBacklogOverflow(entryBytes: number): void {
+    this.overflowed = true;
+    this.droppedPaths += 1;
+    this.droppedBytes += entryBytes;
+    if (this.overflowLogged) return;
+    this.overflowLogged = true;
+    try {
+      this.onBacklogOverflow({
+        pendingPaths: this.pending.size,
+        pendingBytes: this.pendingBytes,
+        maxPendingPaths: this.maxPendingPaths,
+        maxPendingBytes: this.maxPendingBytes,
+        droppedPaths: this.droppedPaths,
+        droppedBytes: this.droppedBytes,
+      });
+    } catch (err) {
+      console.error("TreeWatcher backlog overflow logger error:", err);
+    }
+  }
+
   private arm(): void {
     if (this.timer !== null) {
       this.clock.clearTimeout(this.timer);
@@ -671,7 +673,12 @@ export class TreeWatcher {
     // Snapshot + clear the accumulated paths so the next window starts fresh
     // even if a listener re-enters synchronously.
     const batch: TreeChangeBatch = { paths: new Map(this.pending) };
-    this.pending.clear();
+    if (this.overflowed) {
+      batch.overflowed = true;
+      batch.droppedPaths = this.droppedPaths;
+      batch.droppedBytes = this.droppedBytes;
+    }
+    this.clearPending();
     // First changed relative path of the burst — the US-003 routing argument.
     // undefined when the window settled with no captured path (e.g. a synthetic
     // arm() with no handleEvent).
@@ -685,6 +692,15 @@ export class TreeWatcher {
     }
   }
 
+  private clearPending(): void {
+    this.pending.clear();
+    this.pendingBytes = 0;
+    this.overflowed = false;
+    this.overflowLogged = false;
+    this.droppedPaths = 0;
+    this.droppedBytes = 0;
+  }
+
   /** True while the watch backend is active. */
   isWatching(): boolean {
     return this.backend !== null;
@@ -705,7 +721,7 @@ export class TreeWatcher {
       this.clock.clearTimeout(this.timer);
       this.timer = null;
     }
-    this.pending.clear();
+    this.clearPending();
     if (this.backend) {
       this.backend.close();
       this.backend = null;
@@ -792,6 +808,7 @@ export interface EmitterLogger {
  * running TreeWatcher (returns an unsubscribe fn). Flag-gated + failure-safe.
  */
 export class PushEventEmitter {
+  private static readonly MAX_CONCURRENT_PUBLISHES = 16;
   private readonly originTenantId: string;
   private readonly originDeviceId: string;
   private readonly transport: PushTransport;
@@ -801,6 +818,7 @@ export class PushEventEmitter {
   private readonly logger: EmitterLogger | undefined;
   private internalSeq = 0;
   private readonly nextSeq: () => number;
+  private publishTail: Promise<void> = Promise.resolve();
 
   constructor(opts: PushEventEmitterOptions) {
     this.originTenantId = opts.originTenantId;
@@ -848,12 +866,35 @@ export class PushEventEmitter {
    */
   async emitForBatch(batch: TreeChangeBatch): Promise<void> {
     if (!this.enabled) return;
-    const entries = [...batch.paths.entries()];
-    await Promise.all(
-      entries.map(([absolutePath, relativePath]) =>
-        this.emitOne(absolutePath, relativePath),
-      ),
+    const entriesByRel = new Map<string, [string, string]>();
+    for (const [absolutePath, relativePath] of batch.paths.entries()) {
+      entriesByRel.set(relativePath, [absolutePath, relativePath]);
+    }
+    const entries = [...entriesByRel.values()];
+    const run = this.publishTail.then(
+      () => this.emitEntries(entries),
+      () => this.emitEntries(entries),
     );
+    this.publishTail = run.catch(() => undefined);
+    await run;
+  }
+
+  private async emitEntries(entries: Array<[string, string]>): Promise<void> {
+    for (
+      let i = 0;
+      i < entries.length;
+      i += PushEventEmitter.MAX_CONCURRENT_PUBLISHES
+    ) {
+      const chunk = entries.slice(
+        i,
+        i + PushEventEmitter.MAX_CONCURRENT_PUBLISHES,
+      );
+      await Promise.all(
+        chunk.map(([absolutePath, relativePath]) =>
+          this.emitOne(absolutePath, relativePath),
+        ),
+      );
+    }
   }
 
   private async emitOne(