@indigoai-us/hq-cloud 5.24.0 → 5.26.0

package/src/watcher.ts

@@ -16,9 +16,185 @@ import { createIgnoreFilter, isWithinSizeLimit } from "./ignore.js";
 import { readJournal, writeJournal, hashFile, updateEntry } from "./journal.js";
 import { uploadFile, deleteRemoteFile } from "./s3.js";
 import type { UploadAuthor } from "./s3.js";
+import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
+import { PERSONAL_VAULT_EXCLUDED_TOP_LEVEL } from "./personal-vault.js";
 
 const DEBOUNCE_MS = 2000;
 
+/**
+ * Injectable clock seam (US-001).
+ *
+ * Production code uses the host timers; tests inject a {@link FakeClock} so the
+ * debounce window can be advanced deterministically without real wall-clock
+ * sleeps. US-002 (real chokidar watcher) and US-003 (runner wiring) build on
+ * this same seam — keep the surface minimal and stable.
+ */
+export interface Clock {
+  /** Schedule `fn` to run after `ms`. Returns an opaque handle. */
+  setTimeout(fn: () => void, ms: number): unknown;
+  /** Cancel a previously scheduled timeout. Safe to call with a stale handle. */
+  clearTimeout(handle: unknown): void;
+  /** Current epoch milliseconds. */
+  now(): number;
+}
+
+/** Real clock backed by host timers + Date.now. The production default. */
+export const systemClock: Clock = {
+  setTimeout: (fn, ms) => setTimeout(fn, ms),
+  clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
+  now: () => Date.now(),
+};
+
+interface FakeTimer {
+  id: number;
+  fireAt: number;
+  fn: () => void;
+}
+
+/**
+ * Deterministic clock for tests. Advance virtual time with {@link advance};
+ * any timers whose deadline has passed fire in scheduled order. No real timers
+ * are ever created, so a test using only this clock leaks nothing.
+ */
+export class FakeClock implements Clock {
+  private current = 0;
+  private nextId = 1;
+  private timers = new Map<number, FakeTimer>();
+
+  now(): number {
+    return this.current;
+  }
+
+  setTimeout(fn: () => void, ms: number): unknown {
+    const id = this.nextId++;
+    this.timers.set(id, { id, fireAt: this.current + Math.max(0, ms), fn });
+    return id;
+  }
+
+  clearTimeout(handle: unknown): void {
+    if (typeof handle === "number") this.timers.delete(handle);
+  }
+
+  /**
+   * Advance virtual time by `ms`, firing every timer whose deadline falls in
+   * the interval (in deadline order). Timers scheduled by a firing callback are
+   * honored within the same advance if their new deadline is still within the
+   * advanced window.
+   */
+  advance(ms: number): void {
+    const target = this.current + ms;
+    while (true) {
+      const due = [...this.timers.values()]
+        .filter((t) => t.fireAt <= target)
+        .sort((a, b) => a.fireAt - b.fireAt || a.id - b.id);
+      if (due.length === 0) break;
+      const next = due[0];
+      this.timers.delete(next.id);
+      this.current = Math.max(this.current, next.fireAt);
+      next.fn();
+    }
+    this.current = target;
+  }
+
+  /** Number of timers still pending — a leak check for tests. */
+  pendingTimerCount(): number {
+    return this.timers.size;
+  }
+}
+
+/** A push pass to run when a debounced change settles. May be async. */
+export type PushFn = () => void | Promise<void>;
+
+export interface WatchPushDriverOptions {
+  /** Quiet window (ms) before a settled change triggers a push. */
+  debounceMs?: number;
+  /** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
+  clock?: Clock;
+  /** Push pass to invoke when the window settles. */
+  push: PushFn;
+}
+
+/**
+ * The reusable debounce + coalesce + concurrency-guard core of event-driven
+ * push (US-001 seam).
+ *
+ * It is intentionally decoupled from chokidar and from S3: callers feed it
+ * synthetic or real change notifications via {@link notifyChange}, and it
+ * invokes the injected `push` fn at most once per quiet window. A push that is
+ * still in flight is never overlapped — a change arriving mid-push is collapsed
+ * and re-triggers a single follow-up pass after the next quiet window.
+ *
+ * US-002 wires a real chokidar watcher's events into {@link notifyChange};
+ * US-003 supplies the targeted-push `push` fn. Tests drive it directly with a
+ * {@link FakeClock} and a spy `push` fn (no real S3, no 10-minute sleep).
+ */
+export class WatchPushDriver {
+  private readonly debounceMs: number;
+  private readonly clock: Clock;
+  private readonly push: PushFn;
+  private timer: unknown = null;
+  private pushing = false;
+  private pendingWhilePushing = false;
+  private disposed = false;
+
+  constructor(opts: WatchPushDriverOptions) {
+    this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
+    this.clock = opts.clock ?? systemClock;
+    this.push = opts.push;
+  }
+
+  /**
+   * Register a change. Resets the quiet window; the push fires `debounceMs`
+   * after the LAST change in a burst, coalescing the burst to one push.
+   */
+  notifyChange(): void {
+    if (this.disposed) return;
+    if (this.timer !== null) {
+      this.clock.clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.timer = this.clock.setTimeout(() => {
+      this.timer = null;
+      void this.fire();
+    }, this.debounceMs);
+  }
+
+  private async fire(): Promise<void> {
+    if (this.disposed) return;
+    if (this.pushing) {
+      // A pass is in flight — collapse this trigger; re-arm after it settles.
+      this.pendingWhilePushing = true;
+      return;
+    }
+    this.pushing = true;
+    try {
+      await this.push();
+    } finally {
+      this.pushing = false;
+      if (this.pendingWhilePushing && !this.disposed) {
+        this.pendingWhilePushing = false;
+        // Re-arm a fresh quiet window for the change(s) seen mid-push.
+        this.notifyChange();
+      }
+    }
+  }
+
+  /** True while a push pass is executing. */
+  isPushing(): boolean {
+    return this.pushing;
+  }
+
+  /** Cancel any pending debounce timer; idempotent. Leaves no timers behind. */
+  dispose(): void {
+    this.disposed = true;
+    if (this.timer !== null) {
+      this.clock.clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.pendingWhilePushing = false;
+  }
+}
+
 interface PendingChange {
   type: "add" | "change" | "unlink";
   absolutePath: string;
@@ -146,3 +322,213 @@ export class SyncWatcher {
     }
   }
 }
+
+// ---------------------------------------------------------------------------
+// US-002 — debounced, ignore-aware, exclusion-aware tree watcher.
+//
+// Unlike SyncWatcher (which does S3 work inline), TreeWatcher is a pure change
+// detector: it emits a single debounced `changed` callback after a quiet window
+// and never touches S3 itself. US-003 wires that callback to a targeted push.
+//
+// The emit decision composes the SAME filter stack the push walk uses, so a
+// path that the push would skip never wakes the watcher:
+//   1. createIgnoreFilter  — .hqignore/.gitignore/DEFAULT_IGNORES
+//   2. PERSONAL_VAULT_DEFAULT_EXCLUSIONS (personalMode) — .env/output/.beads/…
+//   3. PERSONAL_VAULT_EXCLUDED_TOP_LEVEL (personalMode) — .git/companies/repos/workspace
+// ---------------------------------------------------------------------------
+
+/** Decision for a single path: emit a change for it, or ignore it. */
+export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
+
+/**
+ * Build the composite emit-decision predicate. Returns true when a change to
+ * `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
+ * layer). Pure and chokidar-free so the matching logic is unit-testable
+ * directly.
+ *
+ * @param hqRoot       sync root (== personal-vault root in personalMode).
+ * @param personalMode when true, also applies the personal-vault default
+ *                     exclusions and the excluded-top-level buckets.
+ */
+export function createWatchPathFilter(
+  hqRoot: string,
+  personalMode = false,
+): WatchPathFilter {
+  const ignoreFilter = createIgnoreFilter(hqRoot);
+  const excludedTopLevel = new Set(PERSONAL_VAULT_EXCLUDED_TOP_LEVEL);
+
+  return (absolutePath: string, isDir = false): boolean => {
+    const rel = path.relative(hqRoot, absolutePath).split(path.sep).join("/");
+    // The root itself, or anything outside it, is never an emit target.
+    if (rel === "" || rel.startsWith("..")) return false;
+
+    // Layer 1: shared ignore stack (.hqignore/.gitignore/DEFAULT_IGNORES).
+    // createIgnoreFilter returns true = "sync this"; false = "ignored".
+    if (!ignoreFilter(absolutePath, isDir)) return false;
+
+    if (personalMode) {
+      // Layer 3: excluded top-level buckets (.git/companies/repos/workspace).
+      const topLevel = rel.split("/")[0];
+      if (excludedTopLevel.has(topLevel)) return false;
+
+      // Layer 2: personal-vault default exclusions (.env/output/.beads/…).
+      if (isPersonalVaultExcluded(rel, isDir)) return false;
+    }
+
+    return true;
+  };
+}
+
+export interface TreeWatcherOptions {
+  /** Sync root to watch (== personal-vault root in personalMode). */
+  hqRoot: string;
+  /** Quiet window (ms) before a settled burst emits one `changed` call. */
+  debounceMs?: number;
+  /** Apply personal-vault default + top-level exclusions. */
+  personalMode?: boolean;
+  /** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
+  clock?: Clock;
+  /**
+   * Pre-built path filter override (test seam). When omitted, one is built
+   * from {@link createWatchPathFilter}.
+   */
+  pathFilter?: WatchPathFilter;
+}
+
+/**
+ * Chokidar-backed file watcher that emits a single debounced `changed` signal
+ * after a {@link debounceMs} quiet window, coalescing bursts. It honors the
+ * full exclusion stack via {@link createWatchPathFilter}, so excluded paths
+ * (`.env`, `output/`, `.git/`, `companies/` in personalMode, …) never emit.
+ *
+ * Lifecycle: {@link start} (idempotent), {@link stop}, {@link dispose}. Stop
+ * closes the chokidar watcher (releasing fds) and cancels any pending debounce
+ * timer. dispose() is stop() + permanent shutdown.
+ */
+export class TreeWatcher {
+  private readonly hqRoot: string;
+  private readonly debounceMs: number;
+  private readonly clock: Clock;
+  private readonly shouldEmit: WatchPathFilter;
+  private watcher: FSWatcher | null = null;
+  private timer: unknown = null;
+  private listeners = new Set<() => void>();
+  private disposed = false;
+
+  constructor(opts: TreeWatcherOptions) {
+    this.hqRoot = opts.hqRoot;
+    this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
+    this.clock = opts.clock ?? systemClock;
+    this.shouldEmit =
+      opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
+  }
+
+  /** Register a debounced-`changed` listener. Returns an unsubscribe fn. */
+  onChange(listener: () => void): () => void {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  }
+
+  /**
+   * Begin watching. Idempotent — a second call while already running is a
+   * no-op (no second chokidar instance, no leaked fds).
+   */
+  start(): void {
+    if (this.disposed || this.watcher) return;
+
+    this.watcher = watch(this.hqRoot, {
+      // chokidar passes a stats hint so dir-only gitignore patterns (`foo/`)
+      // match directory entries during the descent decision. `ignored`
+      // returns true to SKIP, so invert the emit predicate. The watched root
+      // itself must NEVER be ignored (shouldEmit returns false for it because
+      // it is not a valid emit target), or chokidar tears down the whole watch.
+      ignored: (filePath: string, stats?: fs.Stats) => {
+        if (path.resolve(filePath) === path.resolve(this.hqRoot)) return false;
+        return !this.shouldEmit(filePath, stats?.isDirectory());
+      },
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 500,
+        pollInterval: 100,
+      },
+    });
+
+    const onEvent = (absolutePath: string) => this.handleEvent(absolutePath);
+    this.watcher
+      .on("add", onEvent)
+      .on("change", onEvent)
+      .on("unlink", onEvent)
+      .on("addDir", onEvent)
+      .on("unlinkDir", onEvent)
+      .on("error", (err) => console.error("TreeWatcher error:", err));
+  }
+
+  /**
+   * Test/seam entry point: feed a raw filesystem path as if chokidar reported
+   * it. Applies the emit filter then arms the debounce. Real chokidar events
+   * route through here too.
+   */
+  handleEvent(absolutePath: string): void {
+    if (this.disposed) return;
+    // Defensive: chokidar's `ignored` already filtered, but a path that slips
+    // through (or a synthetic test event) is re-checked here.
+    if (!this.shouldEmit(absolutePath, false)) return;
+    this.arm();
+  }
+
+  private arm(): void {
+    if (this.timer !== null) {
+      this.clock.clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.timer = this.clock.setTimeout(() => {
+      this.timer = null;
+      this.emit();
+    }, this.debounceMs);
+  }
+
+  private emit(): void {
+    if (this.disposed) return;
+    for (const l of this.listeners) {
+      try {
+        l();
+      } catch (err) {
+        console.error("TreeWatcher listener error:", err);
+      }
+    }
+  }
+
+  /** True while the chokidar watcher is active. */
+  isWatching(): boolean {
+    return this.watcher !== null;
+  }
+
+  /** Number of pending debounce timers — a leak check for tests. */
+  pendingTimerCount(): number {
+    return this.timer === null ? 0 : 1;
+  }
+
+  /**
+   * Stop watching: close the chokidar watcher (releasing fds) and cancel any
+   * pending debounce timer. Idempotent. The instance can be restarted with
+   * {@link start} unless {@link dispose} was called.
+   */
+  stop(): void {
+    if (this.timer !== null) {
+      this.clock.clearTimeout(this.timer);
+      this.timer = null;
+    }
+    if (this.watcher) {
+      void this.watcher.close();
+      this.watcher = null;
+    }
+  }
+
+  /** Permanent shutdown: stop() + drop listeners; further events are no-ops. */
+  dispose(): void {
+    this.stop();
+    this.disposed = true;
+    this.listeners.clear();
+  }
+}