@indigoai-us/hq-cloud 5.26.0 → 5.28.0

package/src/watcher.ts

@@ -8,6 +8,8 @@
  */
 
 import * as fs from "fs";
+import { createHash } from "node:crypto";
+import { readFile, stat } from "node:fs/promises";
 import * as path from "path";
 import { watch } from "chokidar";
 import type { FSWatcher } from "chokidar";
@@ -18,6 +20,9 @@ import { uploadFile, deleteRemoteFile } from "./s3.js";
 import type { UploadAuthor } from "./s3.js";
 import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
 import { PERSONAL_VAULT_EXCLUDED_TOP_LEVEL } from "./personal-vault.js";
+import type { PushEvent } from "./sync/push-event.js";
+import type { PushTransport } from "./sync/push-transport.js";
+import type { EventDrivenPushFlagProvider } from "./sync/feature-flags.js";
 
 const DEBOUNCE_MS = 2000;
 
@@ -222,10 +227,11 @@ export class SyncWatcher {
     if (this.watcher) return;
 
     this.watcher = watch(this.hqRoot, {
-      // Forward chokidar's stats hint so dir-only gitignore patterns
-      // (`foo/`) match directory entries during the descent decision.
-      ignored: (filePath: string, stats?: fs.Stats) =>
-        !this.shouldSync(filePath, stats?.isDirectory()),
+      // See toChokidarIgnored: chokidar's pre-stat descent probe has no stats
+      // hint, so a naive file-verdict prunes intermediate allowlist dirs
+      // before descending to their in-scope leaves. Keep a statless probe
+      // when EITHER the file or directory reading would survive the filter.
+      ignored: toChokidarIgnored(this.shouldSync, this.hqRoot),
       persistent: true,
       ignoreInitial: true,
       awaitWriteFinish: {
@@ -340,6 +346,131 @@ export class SyncWatcher {
 /** Decision for a single path: emit a change for it, or ignore it. */
 export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
 
+/**
+ * Translate an emit-filter into chokidar's `ignored` predicate (true = SKIP).
+ *
+ * The subtlety this encodes: chokidar probes a candidate path for the descent
+ * decision BEFORE it has stat()ed it, so `stats` is `undefined` on that first
+ * call — and if we say "ignore" there, chokidar prunes the subtree and never
+ * descends. In allowlist mode (`.hqinclude`) the in-scope leaves
+ * (`companies/*\/knowledge/`) live UNDER intermediate dirs (`companies/`,
+ * `companies/<slug>/`) that only survive the filter when queried as a
+ * directory (the ancestor matcher fires for `isDir=true` only). With no stats
+ * hint we don't know if the candidate is that ancestor dir or an in-scope
+ * file, so we must NOT prune when EITHER reading would keep it. The real
+ * isDir-accurate verdict is reapplied at event time (TreeWatcher.handleEvent
+ * re-checks with the known kind), so this only widens descent, never emit.
+ */
+function toChokidarIgnored(
+  shouldEmit: WatchPathFilter,
+  hqRoot: string,
+): (filePath: string, stats?: fs.Stats) => boolean {
+  const root = path.resolve(hqRoot);
+  return (filePath: string, stats?: fs.Stats): boolean => {
+    // The watched root itself must NEVER be ignored, or chokidar tears down
+    // the whole watch. (shouldEmit returns false for it — it's not a valid
+    // emit target — so we special-case it here.)
+    if (path.resolve(filePath) === root) return false;
+    const isDir = stats?.isDirectory();
+    if (isDir === undefined) {
+      // Pre-stat descent probe: keep the path if it could be an in-scope file
+      // OR an ancestor directory of one.
+      return !(shouldEmit(filePath, false) || shouldEmit(filePath, true));
+    }
+    return !shouldEmit(filePath, isDir);
+  };
+}
+
+/** A started watch backend. `close()` releases its OS handle(s); idempotent. */
+interface WatchBackend {
+  close(): void;
+}
+
+/**
+ * Platforms where Node's `fs.watch(dir, { recursive: true })` is natively
+ * supported by a SINGLE OS handle: macOS (FSEvents) and Windows
+ * (ReadDirectoryChangesW). On Linux, recursive `fs.watch` is NOT implemented
+ * (it throws `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM`), so we fall back to
+ * chokidar there.
+ */
+function supportsRecursiveWatch(): boolean {
+  return process.platform === "darwin" || process.platform === "win32";
+}
+
+/**
+ * Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
+ *
+ * Backend selection (the whole point of this helper):
+ *   - macOS / Windows → ONE recursive `fs.watch` handle for the entire tree.
+ *     chokidar 4 dropped its `fsevents` backend, so on macOS it watches via
+ *     kqueue, which costs ~1 open fd PER watched path — ~11k fds over a real
+ *     HQ tree, which EMFILEs under the default soft `ulimit -n` (256) and
+ *     silently kills the watcher (instant sync then falls back to the poll).
+ *     A single recursive watch is 1 handle regardless of tree size.
+ *   - Linux / other → chokidar (recursive `fs.watch` is unsupported there).
+ *
+ * The recursive backend does NO descent-time filtering (it can't — the OS
+ * reports the whole subtree), so EVERY change is funneled to `onEvent`, which
+ * re-applies the emit filter via {@link TreeWatcher.handleEvent}. Excluded
+ * paths (`repos/`, `node_modules/`, out-of-scope companies) are cheaply
+ * dropped there before they arm the debounce. This is also why the recursive
+ * backend is immune to the `.hqinclude` ancestor-descent pruning that the
+ * chokidar backend needs {@link toChokidarIgnored} to avoid.
+ */
+function startTreeWatch(
+  hqRoot: string,
+  shouldEmit: WatchPathFilter,
+  onEvent: (absolutePath: string) => void,
+  onError: (err: unknown) => void,
+): WatchBackend {
+  if (supportsRecursiveWatch()) {
+    try {
+      const native = fs.watch(
+        hqRoot,
+        { recursive: true, persistent: true },
+        (_eventType, filename) => {
+          // `filename` is relative to hqRoot (or null/empty if the platform
+          // couldn't provide it — nothing actionable then). `String()` coerces
+          // both the string and (older @types/node) Buffer shapes.
+          if (filename == null) return;
+          const rel = String(filename);
+          if (rel === "") return;
+          onEvent(path.resolve(hqRoot, rel));
+        },
+      );
+      native.on("error", onError);
+      return { close: () => native.close() };
+    } catch (err) {
+      // Recursive watch unexpectedly unavailable — fall back to chokidar
+      // rather than leaving the daemon with no watcher at all.
+      onError(err);
+    }
+  }
+
+  const cw = watch(hqRoot, {
+    // chokidar fallback (Linux): see toChokidarIgnored for why the descent
+    // probe must not prune ancestor dirs of allowlisted leaves.
+    ignored: toChokidarIgnored(shouldEmit, hqRoot),
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 500,
+      pollInterval: 100,
+    },
+  });
+  cw.on("add", onEvent)
+    .on("change", onEvent)
+    .on("unlink", onEvent)
+    .on("addDir", onEvent)
+    .on("unlinkDir", onEvent)
+    .on("error", onError);
+  return {
+    close: () => {
+      void cw.close();
+    },
+  };
+}
+
 /**
  * Build the composite emit-decision predicate. Returns true when a change to
  * `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
@@ -405,14 +536,43 @@ export interface TreeWatcherOptions {
  * closes the chokidar watcher (releasing fds) and cancels any pending debounce
  * timer. dispose() is stop() + permanent shutdown.
  */
+/**
+ * One settled change-burst, handed to {@link TreeWatcher} listeners. Carries
+ * the set of relative paths that changed during the quiet window so a listener
+ * (e.g. {@link PushEventEmitter}) can build one PushEvent per path. `paths` is
+ * absolute-path → relative-path; both are needed (relative for the wire shape,
+ * absolute for hashing/statting the file on disk).
+ */
+export interface TreeChangeBatch {
+  /** Map of absolutePath → relativePath for every path in the settled burst. */
+  paths: Map<string, string>;
+}
+
+/**
+ * Listener invoked once per settled debounce window.
+ *
+ * Backwards compatible with the US-003 `WatcherSurface` contract: the first
+ * argument is the OPTIONAL changed relative path the loop routes its targeted
+ * push to (the first path of the burst; undefined when the window settled with
+ * no captured path). US-008's {@link PushEventEmitter} consumes the SECOND
+ * argument — the full {@link TreeChangeBatch} of every path in the burst — to
+ * build one PushEvent per path. Listeners are free to ignore either argument.
+ */
+export type TreeChangeListener = (
+  changedRelPath?: string,
+  batch?: TreeChangeBatch,
+) => void;
+
 export class TreeWatcher {
   private readonly hqRoot: string;
   private readonly debounceMs: number;
   private readonly clock: Clock;
   private readonly shouldEmit: WatchPathFilter;
-  private watcher: FSWatcher | null = null;
+  private backend: WatchBackend | null = null;
   private timer: unknown = null;
-  private listeners = new Set<() => void>();
+  private listeners = new Set<TreeChangeListener>();
+  /** Paths accumulated for the current (in-flight) debounce window. */
+  private pending = new Map<string, string>();
   private disposed = false;
 
   constructor(opts: TreeWatcherOptions) {
@@ -423,57 +583,52 @@ export class TreeWatcher {
       opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
   }
 
-  /** Register a debounced-`changed` listener. Returns an unsubscribe fn. */
-  onChange(listener: () => void): () => void {
+  /**
+   * Register a debounced-`changed` listener. Returns an unsubscribe fn.
+   *
+   * Listeners receive a {@link TreeChangeBatch} of the paths that changed in
+   * the settled window. Existing US-003 callers that only need the "something
+   * changed" signal can ignore the argument — the contract is backwards
+   * compatible (a zero-arg callback still type-checks).
+   */
+  onChange(listener: TreeChangeListener): () => void {
     this.listeners.add(listener);
     return () => this.listeners.delete(listener);
   }
 
   /**
    * Begin watching. Idempotent — a second call while already running is a
-   * no-op (no second chokidar instance, no leaked fds).
+   * no-op (no second watch backend, no leaked handles).
+   *
+   * Uses a SINGLE recursive `fs.watch` on macOS/Windows (1 OS handle for the
+   * whole tree) and falls back to chokidar on Linux. See {@link startTreeWatch}
+   * for why per-path watching is avoided (kqueue fd exhaustion → EMFILE).
    */
   start(): void {
-    if (this.disposed || this.watcher) return;
-
-    this.watcher = watch(this.hqRoot, {
-      // chokidar passes a stats hint so dir-only gitignore patterns (`foo/`)
-      // match directory entries during the descent decision. `ignored`
-      // returns true to SKIP, so invert the emit predicate. The watched root
-      // itself must NEVER be ignored (shouldEmit returns false for it because
-      // it is not a valid emit target), or chokidar tears down the whole watch.
-      ignored: (filePath: string, stats?: fs.Stats) => {
-        if (path.resolve(filePath) === path.resolve(this.hqRoot)) return false;
-        return !this.shouldEmit(filePath, stats?.isDirectory());
-      },
-      persistent: true,
-      ignoreInitial: true,
-      awaitWriteFinish: {
-        stabilityThreshold: 500,
-        pollInterval: 100,
-      },
-    });
-
-    const onEvent = (absolutePath: string) => this.handleEvent(absolutePath);
-    this.watcher
-      .on("add", onEvent)
-      .on("change", onEvent)
-      .on("unlink", onEvent)
-      .on("addDir", onEvent)
-      .on("unlinkDir", onEvent)
-      .on("error", (err) => console.error("TreeWatcher error:", err));
+    if (this.disposed || this.backend) return;
+    this.backend = startTreeWatch(
+      this.hqRoot,
+      this.shouldEmit,
+      (absolutePath) => this.handleEvent(absolutePath),
+      (err) => console.error("TreeWatcher error:", err),
+    );
   }
 
   /**
-   * Test/seam entry point: feed a raw filesystem path as if chokidar reported
-   * it. Applies the emit filter then arms the debounce. Real chokidar events
-   * route through here too.
+   * Test/seam entry point: feed a raw filesystem path as if the backend
+   * reported it. Applies the emit filter then arms the debounce. Real watch
+   * events route through here too — and for the recursive backend this is the
+   * ONLY place filtering happens, so out-of-scope paths are dropped here.
    */
   handleEvent(absolutePath: string): void {
     if (this.disposed) return;
-    // Defensive: chokidar's `ignored` already filtered, but a path that slips
-    // through (or a synthetic test event) is re-checked here.
+    // The recursive backend does no descent-time filtering, so this is the
+    // authoritative emit gate; the chokidar backend's `ignored` pre-filters
+    // but a slipped-through (or synthetic test) path is re-checked here too.
     if (!this.shouldEmit(absolutePath, false)) return;
+    const abs = path.resolve(absolutePath);
+    const rel = path.relative(this.hqRoot, abs).split(path.sep).join("/");
+    this.pending.set(abs, rel);
     this.arm();
   }
 
@@ -490,18 +645,26 @@ export class TreeWatcher {
 
   private emit(): void {
     if (this.disposed) return;
+    // Snapshot + clear the accumulated paths so the next window starts fresh
+    // even if a listener re-enters synchronously.
+    const batch: TreeChangeBatch = { paths: new Map(this.pending) };
+    this.pending.clear();
+    // First changed relative path of the burst — the US-003 routing argument.
+    // undefined when the window settled with no captured path (e.g. a synthetic
+    // arm() with no handleEvent).
+    const firstRel = [...batch.paths.values()][0];
     for (const l of this.listeners) {
       try {
-        l();
+        l(firstRel, batch);
       } catch (err) {
         console.error("TreeWatcher listener error:", err);
       }
     }
   }
 
-  /** True while the chokidar watcher is active. */
+  /** True while the watch backend is active. */
   isWatching(): boolean {
-    return this.watcher !== null;
+    return this.backend !== null;
   }
 
   /** Number of pending debounce timers — a leak check for tests. */
@@ -510,18 +673,19 @@ export class TreeWatcher {
   }
 
   /**
-   * Stop watching: close the chokidar watcher (releasing fds) and cancel any
-   * pending debounce timer. Idempotent. The instance can be restarted with
-   * {@link start} unless {@link dispose} was called.
+   * Stop watching: close the watch backend (releasing its OS handle) and
+   * cancel any pending debounce timer. Idempotent. The instance can be
+   * restarted with {@link start} unless {@link dispose} was called.
    */
   stop(): void {
     if (this.timer !== null) {
       this.clock.clearTimeout(this.timer);
       this.timer = null;
     }
-    if (this.watcher) {
-      void this.watcher.close();
-      this.watcher = null;
+    this.pending.clear();
+    if (this.backend) {
+      this.backend.close();
+      this.backend = null;
     }
   }
 
@@ -532,3 +696,196 @@ export class TreeWatcher {
     this.listeners.clear();
   }
 }
+
+// ---------------------------------------------------------------------------
+// US-008 — PushEventEmitter: bridge the TreeWatcher to the Phase 2 transport.
+//
+// On each settled debounce window, build one PushEvent per changed path
+// (sha-256 contentHash, monotonic per-device sequenceNumber, originTenantId)
+// and hand it to the injected PushTransport. Feature-flag gated: when the
+// originTenantId is NOT enabled the emitter is DORMANT — it subscribes to
+// nothing and ships nothing, so the Phase 1 poll-only behavior is unchanged.
+//
+// Failure posture: a transport publish that throws (network/non-2xx/timeout)
+// MUST NOT crash the daemon. The emitter catches per-event publish errors and
+// routes them to `onError` (default: console.error); the existing cadence
+// poll remains the safety net that eventually ships the change.
+// ---------------------------------------------------------------------------
+
+/** Computes the canonical contentHash for a file: `sha256:<64-hex>`. */
+async function computeContentHash(absolutePath: string): Promise<string> {
+  const bytes = await readFile(absolutePath);
+  const hex = createHash("sha256").update(bytes).digest("hex");
+  return `sha256:${hex}`;
+}
+
+export interface PushEventEmitterOptions {
+  /** Tenant identifier stamped onto every PushEvent + checked against the flag. */
+  originTenantId: string;
+  /** Device identifier stamped onto every PushEvent. */
+  originDeviceId: string;
+  /** Transport that ships each PushEvent (US-007 NoopPushTransport / HttpPushTransport). */
+  transport: PushTransport;
+  /**
+   * Feature-flag seam. When `isEnabled(originTenantId)` is false the emitter is
+   * dormant: {@link attach} subscribes nothing and {@link emitForBatch} is a
+   * no-op. Defaults are NOT supplied here — the caller injects an
+   * EventDrivenPushFlagProvider so dormancy is explicit.
+   */
+  flagProvider: EventDrivenPushFlagProvider;
+  /**
+   * Returns the next monotonic sequence number for this device. Default: an
+   * internal counter starting at 0. Inject to persist across daemon restarts.
+   */
+  getSequenceNumber?: () => number;
+  /** Clock for eventTimestamp. Default `() => new Date()`. */
+  now?: () => Date;
+  /**
+   * Where publish failures + hash/stat errors go. Default `console.error`.
+   * Receives the offending PushEvent (when known) so callers can correlate.
+   */
+  onError?: (err: Error, ctx: { relativePath?: string }) => void;
+  /**
+   * Optional structured logger for the US-011 3-log diagnostic chain. When
+   * supplied, the emitter logs `event=watcher.emit` (the 1st correlated link)
+   * carrying the PushEvent's `sequenceNumber` — the same join key stamped by
+   * the server `push.receive` log and the client `fanout.receive` log. Default:
+   * no log (the daemon stays quiet unless wired with a logger).
+   */
+  logger?: EmitterLogger;
+}
+
+/**
+ * Minimal structured logger surface for {@link PushEventEmitter}. A pino
+ * `Logger` (from `./sync/logger.ts`) satisfies this; tests inject a fake.
+ */
+export interface EmitterLogger {
+  info(obj: Record<string, unknown>, msg?: string): void;
+}
+
+/**
+ * Bridges {@link TreeWatcher} change batches to a {@link PushTransport} as
+ * typed PushEvents. Construct once per daemon, then {@link attach} to a
+ * running TreeWatcher (returns an unsubscribe fn). Flag-gated + failure-safe.
+ */
+export class PushEventEmitter {
+  private readonly originTenantId: string;
+  private readonly originDeviceId: string;
+  private readonly transport: PushTransport;
+  private readonly flagProvider: EventDrivenPushFlagProvider;
+  private readonly now: () => Date;
+  private readonly onError: (err: Error, ctx: { relativePath?: string }) => void;
+  private readonly logger: EmitterLogger | undefined;
+  private internalSeq = 0;
+  private readonly nextSeq: () => number;
+
+  constructor(opts: PushEventEmitterOptions) {
+    this.originTenantId = opts.originTenantId;
+    this.originDeviceId = opts.originDeviceId;
+    this.transport = opts.transport;
+    this.flagProvider = opts.flagProvider;
+    this.now = opts.now ?? (() => new Date());
+    this.logger = opts.logger;
+    this.onError =
+      opts.onError ??
+      ((err, ctx) =>
+        console.error("PushEventEmitter error:", ctx.relativePath ?? "", err));
+    this.nextSeq =
+      opts.getSequenceNumber ??
+      (() => {
+        const n = this.internalSeq;
+        this.internalSeq += 1;
+        return n;
+      });
+  }
+
+  /** True iff event-driven push is enabled for this emitter's tenant. */
+  get enabled(): boolean {
+    return this.flagProvider.isEnabled(this.originTenantId);
+  }
+
+  /**
+   * Subscribe to a TreeWatcher's change batches. Dormant (no subscription)
+   * when the flag is OFF for this tenant. Returns an unsubscribe fn (a no-op
+   * when dormant).
+   */
+  attach(watcher: TreeWatcher): () => void {
+    if (!this.enabled) return () => {};
+    return watcher.onChange((_changedRelPath, batch) => {
+      if (batch) void this.emitForBatch(batch);
+    });
+  }
+
+  /**
+   * Build + ship one PushEvent per changed path in the batch. No-op when the
+   * flag is OFF. Each path is independent: a hash/stat failure or a transport
+   * publish rejection for one path is caught + surfaced via `onError` and does
+   * NOT abort the others or propagate (the daemon must not crash; the cadence
+   * poll covers any miss).
+   */
+  async emitForBatch(batch: TreeChangeBatch): Promise<void> {
+    if (!this.enabled) return;
+    const entries = [...batch.paths.entries()];
+    await Promise.all(
+      entries.map(([absolutePath, relativePath]) =>
+        this.emitOne(absolutePath, relativePath),
+      ),
+    );
+  }
+
+  private async emitOne(
+    absolutePath: string,
+    relativePath: string,
+  ): Promise<void> {
+    let event: PushEvent | undefined;
+    try {
+      const [contentHash, st] = await Promise.all([
+        computeContentHash(absolutePath),
+        stat(absolutePath),
+      ]);
+      event = {
+        relativePath,
+        contentHash,
+        mtime: st.mtime.toISOString(),
+        originDeviceId: this.originDeviceId,
+        originTenantId: this.originTenantId,
+        sequenceNumber: this.nextSeq(),
+        eventTimestamp: this.now().toISOString(),
+      };
+    } catch (err) {
+      // File deleted between debounce fire and hash read (legal race), or stat
+      // failed. Surface, don't crash.
+      this.onError(err instanceof Error ? err : new Error(String(err)), {
+        relativePath,
+      });
+      return;
+    }
+
+    // US-011: 1st link of the 3-log diagnostic chain. Stamps the same
+    // `sequenceNumber` the server `push.receive` log and the client
+    // `fanout.receive` log carry, so an operator can walk one event
+    // end-to-end. Logged after the event is built, before the publish.
+    this.logger?.info(
+      {
+        event: "watcher.emit",
+        originTenantId: event.originTenantId,
+        originDeviceId: event.originDeviceId,
+        relativePath: event.relativePath,
+        contentHash: event.contentHash,
+        sequenceNumber: event.sequenceNumber,
+        eventTimestamp: event.eventTimestamp,
+      },
+      "watcher emitted push event",
+    );
+
+    try {
+      await this.transport.publish(event);
+    } catch (err) {
+      // Push failure (network / non-2xx / timeout). The cadence poll is the
+      // safety net — log + continue, never throw.
+      this.onError(err instanceof Error ? err : new Error(String(err)), {
+        relativePath,
+      });
+    }
+  }
+}