@indigoai-us/hq-cloud 5.26.0 → 5.27.0

package/src/sync/push-transport.ts

@@ -31,7 +31,7 @@
  * @indigoai-us/hq-cloud (Path B) per project event-driven-sync-menubar US-007.
  */
 
-import type { PushEvent } from "./push-event.js";
+import { encodePushEvent, type PushEvent } from "./push-event.js";
 
 export interface PushTransport {
   /** Open sockets, refresh tokens. Awaited before the watcher starts. */
@@ -82,3 +82,150 @@ export class NoopPushTransport implements PushTransport {
     this._connected = false;
   }
 }
+
+// ─── HttpPushTransport ───────────────────────────────────────────────────────
+
+/**
+ * Minimal fetch surface so tests can inject a mocked `fetch` without pulling
+ * in DOM/undici types. Matches the subset of the global `fetch` we use.
+ */
+export type FetchLike = (
+  input: string,
+  init?: {
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+    signal?: AbortSignal;
+  },
+) => Promise<{ ok: boolean; status: number; text(): Promise<string> }>;
+
+/**
+ * Async getter for the current Cognito access token. Long-running daemons MUST
+ * pass a getter (not a captured string) so each publish resolves the freshest
+ * token — mirrors {@link VaultServiceConfig.authToken} semantics in types.ts.
+ * A static string is also accepted for short-lived tools and tests.
+ */
+export type AuthTokenSource = string | (() => string | Promise<string>);
+
+export interface HttpPushTransportOptions {
+  /**
+   * Vault API base URL (e.g. `https://vault-api.example.com`). The same
+   * `apiUrl` the runner already resolves for VaultClient. Trailing slashes
+   * are stripped. Required — config/env-driven, no hard-coded default.
+   */
+  apiUrl: string;
+  /**
+   * Endpoint path the PushEvent is POSTed to. Defaults to `/sync/push`
+   * (matches the deployed hq-pro endpoint from US-006). Override for testing
+   * or future endpoint moves.
+   */
+  pushPath?: string;
+  /** Cognito JWT — static string OR async getter. See {@link AuthTokenSource}. */
+  authToken: AuthTokenSource;
+  /**
+   * Optional extra headers (e.g. client identification) merged onto every
+   * request. Authorization + Content-Type are always set by the transport.
+   */
+  headers?: Record<string, string>;
+  /**
+   * Per-request timeout in milliseconds. Default 10_000. On timeout the
+   * publish rejects — the daemon treats a rejected publish as a transient
+   * miss (the cadence poll still covers it) and MUST NOT crash.
+   */
+  timeoutMs?: number;
+  /** Injectable fetch (tests). Defaults to the global `fetch`. */
+  fetchImpl?: FetchLike;
+}
+
+/**
+ * Real client `PushTransport` that POSTs encoded PushEvents to the deployed
+ * `/sync/push` endpoint, authenticating with the menubar's existing Cognito
+ * bearer token — the same auth path VaultClient uses (Authorization: Bearer
+ * <accessToken>, token resolved per-request via the supplied getter so it
+ * self-heals across refreshes).
+ *
+ * Failure posture
+ * ───────────────
+ * `publish()` rejects on a network error, a non-2xx response, or a timeout.
+ * The DAEMON is responsible for not letting that rejection crash it — the
+ * watcher's emit path catches publish errors and logs them; the periodic
+ * cadence poll remains the safety net that eventually ships the change. This
+ * transport never swallows errors itself, so callers retain full visibility.
+ *
+ * `connected` flips true on `start()` and false on `dispose()`. It is purely
+ * advisory (HTTP is connectionless); the health endpoint may read it but it
+ * carries no delivery guarantee.
+ */
+export class HttpPushTransport implements PushTransport {
+  private readonly apiUrl: string;
+  private readonly pushPath: string;
+  private readonly getAuthToken: () => Promise<string>;
+  private readonly extraHeaders: Record<string, string>;
+  private readonly timeoutMs: number;
+  private readonly fetchImpl: FetchLike;
+  private _connected = false;
+
+  constructor(opts: HttpPushTransportOptions) {
+    if (!opts.apiUrl || opts.apiUrl.trim() === "") {
+      throw new Error("HttpPushTransport: apiUrl is required");
+    }
+    this.apiUrl = opts.apiUrl.replace(/\/+$/, "");
+    const path = opts.pushPath ?? "/sync/push";
+    this.pushPath = path.startsWith("/") ? path : `/${path}`;
+    const tok = opts.authToken;
+    // Normalize string|getter into a single async getter (mirrors VaultClient).
+    this.getAuthToken =
+      typeof tok === "function" ? async () => tok() : async () => tok;
+    this.extraHeaders = opts.headers ?? {};
+    this.timeoutMs = opts.timeoutMs ?? 10_000;
+    // Bind so a destructured global fetch keeps its receiver.
+    this.fetchImpl =
+      opts.fetchImpl ??
+      ((input, init) => (globalThis.fetch as unknown as FetchLike)(input, init));
+  }
+
+  get connected(): boolean {
+    return this._connected;
+  }
+
+  async start(): Promise<void> {
+    this._connected = true;
+  }
+
+  async publish(event: PushEvent): Promise<void> {
+    // Validate-on-encode (US-007 contract) — a malformed event throws a typed
+    // PushEventDecodeError BEFORE we hit the network.
+    const body = encodePushEvent(event);
+    const token = await this.getAuthToken();
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+    try {
+      const res = await this.fetchImpl(`${this.apiUrl}${this.pushPath}`, {
+        method: "POST",
+        headers: {
+          ...this.extraHeaders,
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+          Accept: "application/json",
+        },
+        body,
+        signal: controller.signal,
+      });
+      if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        throw new Error(
+          `HttpPushTransport: POST ${this.pushPath} failed (${res.status})${
+            text ? `: ${text}` : ""
+          }`,
+        );
+      }
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async dispose(): Promise<void> {
+    this._connected = false;
+  }
+}

package/src/watcher.ts

@@ -8,6 +8,8 @@
  */
 
 import * as fs from "fs";
+import { createHash } from "node:crypto";
+import { readFile, stat } from "node:fs/promises";
 import * as path from "path";
 import { watch } from "chokidar";
 import type { FSWatcher } from "chokidar";
@@ -18,6 +20,9 @@ import { uploadFile, deleteRemoteFile } from "./s3.js";
 import type { UploadAuthor } from "./s3.js";
 import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
 import { PERSONAL_VAULT_EXCLUDED_TOP_LEVEL } from "./personal-vault.js";
+import type { PushEvent } from "./sync/push-event.js";
+import type { PushTransport } from "./sync/push-transport.js";
+import type { EventDrivenPushFlagProvider } from "./sync/feature-flags.js";
 
 const DEBOUNCE_MS = 2000;
 
@@ -222,10 +227,11 @@ export class SyncWatcher {
     if (this.watcher) return;
 
     this.watcher = watch(this.hqRoot, {
-      // Forward chokidar's stats hint so dir-only gitignore patterns
-      // (`foo/`) match directory entries during the descent decision.
-      ignored: (filePath: string, stats?: fs.Stats) =>
-        !this.shouldSync(filePath, stats?.isDirectory()),
+      // See toChokidarIgnored: chokidar's pre-stat descent probe has no stats
+      // hint, so a naive file-verdict prunes intermediate allowlist dirs
+      // before descending to their in-scope leaves. Keep a statless probe
+      // when EITHER the file or directory reading would survive the filter.
+      ignored: toChokidarIgnored(this.shouldSync, this.hqRoot),
       persistent: true,
       ignoreInitial: true,
       awaitWriteFinish: {
@@ -340,6 +346,41 @@ export class SyncWatcher {
 /** Decision for a single path: emit a change for it, or ignore it. */
 export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
 
+/**
+ * Translate an emit-filter into chokidar's `ignored` predicate (true = SKIP).
+ *
+ * The subtlety this encodes: chokidar probes a candidate path for the descent
+ * decision BEFORE it has stat()ed it, so `stats` is `undefined` on that first
+ * call — and if we say "ignore" there, chokidar prunes the subtree and never
+ * descends. In allowlist mode (`.hqinclude`) the in-scope leaves
+ * (`companies/*\/knowledge/`) live UNDER intermediate dirs (`companies/`,
+ * `companies/<slug>/`) that only survive the filter when queried as a
+ * directory (the ancestor matcher fires for `isDir=true` only). With no stats
+ * hint we don't know if the candidate is that ancestor dir or an in-scope
+ * file, so we must NOT prune when EITHER reading would keep it. The real
+ * isDir-accurate verdict is reapplied at event time (TreeWatcher.handleEvent
+ * re-checks with the known kind), so this only widens descent, never emit.
+ */
+function toChokidarIgnored(
+  shouldEmit: WatchPathFilter,
+  hqRoot: string,
+): (filePath: string, stats?: fs.Stats) => boolean {
+  const root = path.resolve(hqRoot);
+  return (filePath: string, stats?: fs.Stats): boolean => {
+    // The watched root itself must NEVER be ignored, or chokidar tears down
+    // the whole watch. (shouldEmit returns false for it — it's not a valid
+    // emit target — so we special-case it here.)
+    if (path.resolve(filePath) === root) return false;
+    const isDir = stats?.isDirectory();
+    if (isDir === undefined) {
+      // Pre-stat descent probe: keep the path if it could be an in-scope file
+      // OR an ancestor directory of one.
+      return !(shouldEmit(filePath, false) || shouldEmit(filePath, true));
+    }
+    return !shouldEmit(filePath, isDir);
+  };
+}
+
 /**
  * Build the composite emit-decision predicate. Returns true when a change to
  * `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
@@ -405,6 +446,33 @@ export interface TreeWatcherOptions {
  * closes the chokidar watcher (releasing fds) and cancels any pending debounce
  * timer. dispose() is stop() + permanent shutdown.
  */
+/**
+ * One settled change-burst, handed to {@link TreeWatcher} listeners. Carries
+ * the set of relative paths that changed during the quiet window so a listener
+ * (e.g. {@link PushEventEmitter}) can build one PushEvent per path. `paths` is
+ * absolute-path → relative-path; both are needed (relative for the wire shape,
+ * absolute for hashing/statting the file on disk).
+ */
+export interface TreeChangeBatch {
+  /** Map of absolutePath → relativePath for every path in the settled burst. */
+  paths: Map<string, string>;
+}
+
+/**
+ * Listener invoked once per settled debounce window.
+ *
+ * Backwards compatible with the US-003 `WatcherSurface` contract: the first
+ * argument is the OPTIONAL changed relative path the loop routes its targeted
+ * push to (the first path of the burst; undefined when the window settled with
+ * no captured path). US-008's {@link PushEventEmitter} consumes the SECOND
+ * argument — the full {@link TreeChangeBatch} of every path in the burst — to
+ * build one PushEvent per path. Listeners are free to ignore either argument.
+ */
+export type TreeChangeListener = (
+  changedRelPath?: string,
+  batch?: TreeChangeBatch,
+) => void;
+
 export class TreeWatcher {
   private readonly hqRoot: string;
   private readonly debounceMs: number;
@@ -412,7 +480,9 @@ export class TreeWatcher {
   private readonly shouldEmit: WatchPathFilter;
   private watcher: FSWatcher | null = null;
   private timer: unknown = null;
-  private listeners = new Set<() => void>();
+  private listeners = new Set<TreeChangeListener>();
+  /** Paths accumulated for the current (in-flight) debounce window. */
+  private pending = new Map<string, string>();
   private disposed = false;
 
   constructor(opts: TreeWatcherOptions) {
@@ -423,8 +493,15 @@ export class TreeWatcher {
       opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
   }
 
-  /** Register a debounced-`changed` listener. Returns an unsubscribe fn. */
-  onChange(listener: () => void): () => void {
+  /**
+   * Register a debounced-`changed` listener. Returns an unsubscribe fn.
+   *
+   * Listeners receive a {@link TreeChangeBatch} of the paths that changed in
+   * the settled window. Existing US-003 callers that only need the "something
+   * changed" signal can ignore the argument — the contract is backwards
+   * compatible (a zero-arg callback still type-checks).
+   */
+  onChange(listener: TreeChangeListener): () => void {
     this.listeners.add(listener);
     return () => this.listeners.delete(listener);
   }
@@ -437,15 +514,15 @@ export class TreeWatcher {
     if (this.disposed || this.watcher) return;
 
     this.watcher = watch(this.hqRoot, {
-      // chokidar passes a stats hint so dir-only gitignore patterns (`foo/`)
-      // match directory entries during the descent decision. `ignored`
-      // returns true to SKIP, so invert the emit predicate. The watched root
-      // itself must NEVER be ignored (shouldEmit returns false for it because
-      // it is not a valid emit target), or chokidar tears down the whole watch.
-      ignored: (filePath: string, stats?: fs.Stats) => {
-        if (path.resolve(filePath) === path.resolve(this.hqRoot)) return false;
-        return !this.shouldEmit(filePath, stats?.isDirectory());
-      },
+      // `ignored` returns true to SKIP. chokidar probes a candidate for the
+      // descent decision BEFORE stat()ing it (stats undefined on that call),
+      // so a naive `!shouldEmit(path, stats?.isDirectory())` treats the
+      // statless probe as a file and prunes intermediate dirs (`companies/`)
+      // that only survive the allowlist filter as directories — starving the
+      // in-scope leaves below them. toChokidarIgnored keeps a statless probe
+      // when EITHER the file or the directory reading would; the accurate
+      // isDir verdict is reapplied at event time in handleEvent.
+      ignored: toChokidarIgnored(this.shouldEmit, this.hqRoot),
       persistent: true,
       ignoreInitial: true,
       awaitWriteFinish: {
@@ -474,6 +551,9 @@ export class TreeWatcher {
     // Defensive: chokidar's `ignored` already filtered, but a path that slips
     // through (or a synthetic test event) is re-checked here.
     if (!this.shouldEmit(absolutePath, false)) return;
+    const abs = path.resolve(absolutePath);
+    const rel = path.relative(this.hqRoot, abs).split(path.sep).join("/");
+    this.pending.set(abs, rel);
     this.arm();
   }
 
@@ -490,9 +570,17 @@ export class TreeWatcher {
 
   private emit(): void {
     if (this.disposed) return;
+    // Snapshot + clear the accumulated paths so the next window starts fresh
+    // even if a listener re-enters synchronously.
+    const batch: TreeChangeBatch = { paths: new Map(this.pending) };
+    this.pending.clear();
+    // First changed relative path of the burst — the US-003 routing argument.
+    // undefined when the window settled with no captured path (e.g. a synthetic
+    // arm() with no handleEvent).
+    const firstRel = [...batch.paths.values()][0];
     for (const l of this.listeners) {
       try {
-        l();
+        l(firstRel, batch);
       } catch (err) {
         console.error("TreeWatcher listener error:", err);
       }
@@ -519,6 +607,7 @@ export class TreeWatcher {
       this.clock.clearTimeout(this.timer);
       this.timer = null;
     }
+    this.pending.clear();
     if (this.watcher) {
       void this.watcher.close();
       this.watcher = null;
@@ -532,3 +621,196 @@ export class TreeWatcher {
     this.listeners.clear();
   }
 }
+
+// ---------------------------------------------------------------------------
+// US-008 — PushEventEmitter: bridge the TreeWatcher to the Phase 2 transport.
+//
+// On each settled debounce window, build one PushEvent per changed path
+// (sha-256 contentHash, monotonic per-device sequenceNumber, originTenantId)
+// and hand it to the injected PushTransport. Feature-flag gated: when the
+// originTenantId is NOT enabled the emitter is DORMANT — it subscribes to
+// nothing and ships nothing, so the Phase 1 poll-only behavior is unchanged.
+//
+// Failure posture: a transport publish that throws (network/non-2xx/timeout)
+// MUST NOT crash the daemon. The emitter catches per-event publish errors and
+// routes them to `onError` (default: console.error); the existing cadence
+// poll remains the safety net that eventually ships the change.
+// ---------------------------------------------------------------------------
+
+/** Computes the canonical contentHash for a file: `sha256:<64-hex>`. */
+async function computeContentHash(absolutePath: string): Promise<string> {
+  const bytes = await readFile(absolutePath);
+  const hex = createHash("sha256").update(bytes).digest("hex");
+  return `sha256:${hex}`;
+}
+
+export interface PushEventEmitterOptions {
+  /** Tenant identifier stamped onto every PushEvent + checked against the flag. */
+  originTenantId: string;
+  /** Device identifier stamped onto every PushEvent. */
+  originDeviceId: string;
+  /** Transport that ships each PushEvent (US-007 NoopPushTransport / HttpPushTransport). */
+  transport: PushTransport;
+  /**
+   * Feature-flag seam. When `isEnabled(originTenantId)` is false the emitter is
+   * dormant: {@link attach} subscribes nothing and {@link emitForBatch} is a
+   * no-op. Defaults are NOT supplied here — the caller injects an
+   * EventDrivenPushFlagProvider so dormancy is explicit.
+   */
+  flagProvider: EventDrivenPushFlagProvider;
+  /**
+   * Returns the next monotonic sequence number for this device. Default: an
+   * internal counter starting at 0. Inject to persist across daemon restarts.
+   */
+  getSequenceNumber?: () => number;
+  /** Clock for eventTimestamp. Default `() => new Date()`. */
+  now?: () => Date;
+  /**
+   * Where publish failures + hash/stat errors go. Default `console.error`.
+   * Receives the offending PushEvent (when known) so callers can correlate.
+   */
+  onError?: (err: Error, ctx: { relativePath?: string }) => void;
+  /**
+   * Optional structured logger for the US-011 3-log diagnostic chain. When
+   * supplied, the emitter logs `event=watcher.emit` (the 1st correlated link)
+   * carrying the PushEvent's `sequenceNumber` — the same join key stamped by
+   * the server `push.receive` log and the client `fanout.receive` log. Default:
+   * no log (the daemon stays quiet unless wired with a logger).
+   */
+  logger?: EmitterLogger;
+}
+
+/**
+ * Minimal structured logger surface for {@link PushEventEmitter}. A pino
+ * `Logger` (from `./sync/logger.ts`) satisfies this; tests inject a fake.
+ */
+export interface EmitterLogger {
+  info(obj: Record<string, unknown>, msg?: string): void;
+}
+
+/**
+ * Bridges {@link TreeWatcher} change batches to a {@link PushTransport} as
+ * typed PushEvents. Construct once per daemon, then {@link attach} to a
+ * running TreeWatcher (returns an unsubscribe fn). Flag-gated + failure-safe.
+ */
+export class PushEventEmitter {
+  private readonly originTenantId: string;
+  private readonly originDeviceId: string;
+  private readonly transport: PushTransport;
+  private readonly flagProvider: EventDrivenPushFlagProvider;
+  private readonly now: () => Date;
+  private readonly onError: (err: Error, ctx: { relativePath?: string }) => void;
+  private readonly logger: EmitterLogger | undefined;
+  private internalSeq = 0;
+  private readonly nextSeq: () => number;
+
+  constructor(opts: PushEventEmitterOptions) {
+    this.originTenantId = opts.originTenantId;
+    this.originDeviceId = opts.originDeviceId;
+    this.transport = opts.transport;
+    this.flagProvider = opts.flagProvider;
+    this.now = opts.now ?? (() => new Date());
+    this.logger = opts.logger;
+    this.onError =
+      opts.onError ??
+      ((err, ctx) =>
+        console.error("PushEventEmitter error:", ctx.relativePath ?? "", err));
+    this.nextSeq =
+      opts.getSequenceNumber ??
+      (() => {
+        const n = this.internalSeq;
+        this.internalSeq += 1;
+        return n;
+      });
+  }
+
+  /** True iff event-driven push is enabled for this emitter's tenant. */
+  get enabled(): boolean {
+    return this.flagProvider.isEnabled(this.originTenantId);
+  }
+
+  /**
+   * Subscribe to a TreeWatcher's change batches. Dormant (no subscription)
+   * when the flag is OFF for this tenant. Returns an unsubscribe fn (a no-op
+   * when dormant).
+   */
+  attach(watcher: TreeWatcher): () => void {
+    if (!this.enabled) return () => {};
+    return watcher.onChange((_changedRelPath, batch) => {
+      if (batch) void this.emitForBatch(batch);
+    });
+  }
+
+  /**
+   * Build + ship one PushEvent per changed path in the batch. No-op when the
+   * flag is OFF. Each path is independent: a hash/stat failure or a transport
+   * publish rejection for one path is caught + surfaced via `onError` and does
+   * NOT abort the others or propagate (the daemon must not crash; the cadence
+   * poll covers any miss).
+   */
+  async emitForBatch(batch: TreeChangeBatch): Promise<void> {
+    if (!this.enabled) return;
+    const entries = [...batch.paths.entries()];
+    await Promise.all(
+      entries.map(([absolutePath, relativePath]) =>
+        this.emitOne(absolutePath, relativePath),
+      ),
+    );
+  }
+
+  private async emitOne(
+    absolutePath: string,
+    relativePath: string,
+  ): Promise<void> {
+    let event: PushEvent | undefined;
+    try {
+      const [contentHash, st] = await Promise.all([
+        computeContentHash(absolutePath),
+        stat(absolutePath),
+      ]);
+      event = {
+        relativePath,
+        contentHash,
+        mtime: st.mtime.toISOString(),
+        originDeviceId: this.originDeviceId,
+        originTenantId: this.originTenantId,
+        sequenceNumber: this.nextSeq(),
+        eventTimestamp: this.now().toISOString(),
+      };
+    } catch (err) {
+      // File deleted between debounce fire and hash read (legal race), or stat
+      // failed. Surface, don't crash.
+      this.onError(err instanceof Error ? err : new Error(String(err)), {
+        relativePath,
+      });
+      return;
+    }
+
+    // US-011: 1st link of the 3-log diagnostic chain. Stamps the same
+    // `sequenceNumber` the server `push.receive` log and the client
+    // `fanout.receive` log carry, so an operator can walk one event
+    // end-to-end. Logged after the event is built, before the publish.
+    this.logger?.info(
+      {
+        event: "watcher.emit",
+        originTenantId: event.originTenantId,
+        originDeviceId: event.originDeviceId,
+        relativePath: event.relativePath,
+        contentHash: event.contentHash,
+        sequenceNumber: event.sequenceNumber,
+        eventTimestamp: event.eventTimestamp,
+      },
+      "watcher emitted push event",
+    );
+
+    try {
+      await this.transport.publish(event);
+    } catch (err) {
+      // Push failure (network / non-2xx / timeout). The cadence poll is the
+      // safety net — log + continue, never throw.
+      this.onError(err instanceof Error ? err : new Error(String(err)), {
+        relativePath,
+      });
+    }
+  }
+}