@indigoai-us/hq-cloud 5.26.0 → 5.28.0

package/src/sync/push-receiver.ts

@@ -0,0 +1,1077 @@
+/**
+ * PushReceiver — inbound subscription seam for the hq-cloud watcher daemon
+ * (project event-driven-sync-menubar US-009).
+ *
+ * Mirrors {@link PushTransport} (`./push-transport.ts`) but for the opposite
+ * direction of travel: where the transport SHIPS local file changes out to the
+ * cloud, the receiver SUBSCRIBES to the tenant fanout and triggers an
+ * immediate, TARGETED local pull the moment a peer device of the same tenant
+ * publishes a change. Together they form the event-driven primary path; the
+ * existing `--poll-remote-ms` poll in `runRunnerWithLoop` is the safety net
+ * behind both.
+ *
+ * Transport: SNS → per-client SQS (US-000 decision)
+ * ─────────────────────────────────────────────────
+ * Per the US-000 transport investigation (companies/indigo/projects/
+ * event-driven-sync-menubar/references.md): reuse PR #112's SNS publish +
+ * DynamoDB catch-up log, and build the client RECEIVE side as a per-client
+ * SQS queue subscribed to `sync-push-{tenantId}`. The receiver long-polls its
+ * own queue, decodes each message body as a {@link PushEvent}, dedupes by
+ * `sequenceNumber` per `relativePath`, and bridges into the existing sync
+ * engine via an injected {@link SyncEngineFn} (→ targeted `runRunner` pull).
+ *
+ * The live queue is NOT provisioned yet (the server SQS-provisioning Lambda is
+ * an unbuilt follow-up — see references.md "Open items handed to the plan").
+ * So this module ships:
+ *  - {@link SqsClientLike} — the narrow SQS surface the receiver depends on
+ *    (`receiveMessage` / `deleteMessage`). Production callers pass an
+ *    `@aws-sdk/client-sqs` `SQSClient` adapted to this shape; unit tests inject
+ *    a fake. NO real AWS is required to exercise the receiver.
+ *  - {@link SqsPushReceiver} — the real receiver. Long-polls the queue,
+ *    dispatches each event through the shared dedupe path, deletes the message
+ *    on successful handoff, and reconnects on transient `receiveMessage`
+ *    failures with exponential backoff. SQS's own 14-day retention buffers
+ *    messages while the device is offline → reconnect-replay is "free": on
+ *    reconnect the poll loop simply resumes and the retained messages are
+ *    redelivered, then dedupe skips anything already processed.
+ *  - {@link NoopPushReceiver} — the dormant default. Flips `connected` on
+ *    start, opens no subscription. Wired when the daemon runs without a real
+ *    queue (or when the feature flag is OFF).
+ *  - {@link createPushReceiver} — factory the daemon uses; returns the noop
+ *    unless an SQS client + queue URL are supplied.
+ *
+ * Lifecycle (mirrors PushTransport)
+ * ─────────────────────────────────
+ *  - `start()` opens the subscription (begins the long-poll loop). Awaited
+ *    AFTER the watcher starts so a synthetic event can't race a half-built
+ *    daemon. When the feature flag is OFF, `start()` is a no-op and `connected`
+ *    stays false — NO queue is polled (dormant; AC#4).
+ *  - On each received message: validate with {@link decodePushEvent} (defense
+ *    in depth at the wire boundary), dedupe by `relativePath` against the
+ *    highest `sequenceNumber` seen for that path, then call the injected
+ *    {@link SyncEngineFn}. The sync engine is an injected seam — this story
+ *    does NOT re-implement download logic; it bridges to `runRunner` pull.
+ *  - `dispose()` aborts in-flight via AbortController, stops the poll loop,
+ *    awaits the in-flight sync up to a drain deadline, then disconnects.
+ *
+ * Dedupe (AC#3)
+ * ─────────────
+ * A per-`relativePath` map of the highest `sequenceNumber` already passed to
+ * `syncFn`. An incoming event with `sequenceNumber <= seen` is skipped. SQS
+ * at-least-once delivery + reconnect-replay means the SAME event can arrive
+ * twice; dedupe makes that idempotent.
+ *
+ * Disconnect / reconnect with catch-up replay (AC#3/#4)
+ * ─────────────────────────────────────────────────────
+ * `receiveMessage` failures (network blip, throttling) are caught; the loop
+ * backs off (exponential + jitter, capped) and resumes. Because the per-client
+ * SQS queue retains undelivered messages for 14 days, anything published while
+ * the device was offline/disconnected is redelivered when the poll resumes —
+ * catch-up replay with no server round-trip. Redelivered-but-already-processed
+ * events are absorbed by the dedupe path. The in-memory fake's
+ * `simulateDisconnect()` / `simulateReconnect()` model exactly this buffering.
+ *
+ * Feature flag (AC#4)
+ * ───────────────────
+ * Gated by the per-tenant {@link EventDrivenPushFlagProvider} (US-008's
+ * feature-flags.ts) — honors `HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED_TENANTS`
+ * (per-tenant) + the legacy global `HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED=true`.
+ * Default DISABLED. Resolution precedence: explicit `enabled` > injected
+ * `flagProvider.isEnabled(tenantId)` > default env-driven provider.
+ *
+ * Cross-tenant isolation (US-010)
+ * ───────────────────────────────
+ * Each receiver instance binds exactly ONE `tenantId` and polls exactly ONE
+ * queue URL (its own tenant's per-client queue). Isolation is enforced at the
+ * subscription boundary — the receiver never reads another tenant's queue, and
+ * never filters cross-tenant data post-hoc.
+ *
+ * @see ./push-transport.ts (the outbound seam this mirrors)
+ * @see ./feature-flags.ts (the per-tenant flag provider — US-008)
+ * @see ../bin/sync-runner.ts (the wiring site — runRunnerWithLoop)
+ * @see companies/indigo/projects/event-driven-sync-menubar/references.md (US-000)
+ *
+ * Adapted from indigoai-us/hq-pro PR #112 (src/sync/push-receiver.ts) into
+ * @indigoai-us/hq-cloud (Path B). The hq-pro source shipped only Noop +
+ * InMemory receivers (the production SQS path was deferred there); this module
+ * builds the real SQS receiver behind the same lifecycle/dedupe/flag seam.
+ */
+
+import { decodePushEvent, type PushEvent } from "./push-event.js";
+import {
+  defaultFlagProvider,
+  type EventDrivenPushFlagProvider,
+} from "./feature-flags.js";
+import {
+  publishSyncLatencyMetric,
+  type SyncLatencyMetric,
+} from "./metrics.js";
+
+// ─── Constants ─────────────────────────────────────────────────────────────
+
+/**
+ * How long `dispose()` awaits an in-flight `syncFn` after aborting its signal,
+ * before abandoning it (the poll/cadence safety net re-pulls on the next tick).
+ */
+export const DEFAULT_RECEIVER_DISPOSE_DRAIN_MS = 5_000;
+
+/** Default SQS long-poll wait (seconds). 20 is the SQS max — true long-poll. */
+export const DEFAULT_WAIT_TIME_SECONDS = 20;
+
+/** Default max messages pulled per `receiveMessage` call (SQS max is 10). */
+export const DEFAULT_MAX_MESSAGES = 10;
+
+/** Reconnect backoff defaults. */
+export const DEFAULT_RECONNECT_INITIAL_MS = 250;
+export const DEFAULT_RECONNECT_MAX_MS = 30_000;
+
+// ─── Narrow SQS surface (the injectable transport seam) ──────────────────────
+
+/**
+ * One SQS message as the receiver consumes it. A structural subset of the AWS
+ * SDK `Message` so a real `SQSClient` response satisfies it without adaptation
+ * and tests can build literals.
+ */
+export interface SqsMessageLike {
+  /** The message payload — a JSON-encoded {@link PushEvent}. */
+  readonly Body?: string;
+  /** Opaque handle used to delete the message after successful handoff. */
+  readonly ReceiptHandle?: string;
+  /** Optional SQS message id (logged for diagnostics). */
+  readonly MessageId?: string;
+}
+
+/**
+ * The narrow SQS client surface the receiver depends on. The AWS SDK
+ * `SQSClient` does NOT match this shape directly (it exposes a single
+ * `send(command)`); production callers adapt it with a thin wrapper (see
+ * {@link sqsClientFromAwsSdk} in the wiring site / tests). Keeping the seam
+ * this narrow means unit tests inject a hand-written fake with zero AWS deps.
+ */
+export interface SqsClientLike {
+  /**
+   * Long-poll the queue. Resolves with zero or more messages. MUST honor the
+   * abort signal (resolve/reject promptly on abort) so `dispose()` doesn't
+   * block on an in-flight 20s long-poll.
+   */
+  receiveMessage(args: {
+    queueUrl: string;
+    maxMessages: number;
+    waitTimeSeconds: number;
+    signal: AbortSignal;
+  }): Promise<{ messages: SqsMessageLike[] }>;
+
+  /** Delete a successfully-handled message so it isn't redelivered. */
+  deleteMessage(args: {
+    queueUrl: string;
+    receiptHandle: string;
+  }): Promise<void>;
+}
+
+// ─── Public types ──────────────────────────────────────────────────────────
+
+/**
+ * Context handed to {@link SyncEngineFn} on every received event.
+ *
+ * - `event` — the validated, deduped PushEvent. `relativePath` is what the
+ *   sync engine pulls; `sequenceNumber` is for observability.
+ * - `signal` — aborts when `dispose()` runs past its drain deadline. A
+ *   well-behaved sync fn checks `signal.aborted` between stages and returns
+ *   early.
+ */
+export interface PushReceiverContext {
+  readonly event: PushEvent;
+  readonly signal: AbortSignal;
+}
+
+/**
+ * The injected sync function. The receiver does NOT perform the actual fetch —
+ * it hands off the relativePath to whatever the deployment supplies. In
+ * production this bridges to a targeted `runRunner` pull for the affected
+ * company/path; in tests it's a fake recording invocations.
+ *
+ * Errors from `syncFn` are CAUGHT by the receiver — they log and the loop
+ * continues. A misbehaving sync fn cannot crash the receiver. The poll/cadence
+ * safety net is the recovery path for a thrown pull.
+ */
+export type SyncEngineFn = (ctx: PushReceiverContext) => Promise<void>;
+
+/**
+ * Best-effort CloudWatch metric publish seam (US-011). Invoked on the
+ * receive-SUCCESS path with the measured save-on-A → visible-on-B latency.
+ * Defaults to {@link publishSyncLatencyMetric} (the module singleton client);
+ * tests inject a spy so no real AWS is touched. The receiver awaits it inside a
+ * try/catch — a metric failure can never crash the loop (it's also best-effort
+ * inside the default impl).
+ */
+export type PublishMetricFn = (metric: SyncLatencyMetric) => Promise<void>;
+
+/** Minimal structured logger. Defaults to a no-op (quiet daemon). */
+export interface ReceiverLogger {
+  info(obj: Record<string, unknown>, msg?: string): void;
+  warn(obj: Record<string, unknown>, msg?: string): void;
+  error(obj: Record<string, unknown>, msg?: string): void;
+  debug(obj: Record<string, unknown>, msg?: string): void;
+}
+
+const NOOP_LOGGER: ReceiverLogger = {
+  info: () => undefined,
+  warn: () => undefined,
+  error: () => undefined,
+  debug: () => undefined,
+};
+
+/**
+ * Lifecycle handle. Mirrors {@link PushTransport} so daemon wiring is
+ * mechanically identical on both sides.
+ */
+export interface PushReceiver {
+  /** Open the subscription / poll loop. No-op when the feature flag is OFF. */
+  start(): Promise<void>;
+  /** Idempotent teardown — stop polling, abort + drain in-flight, disconnect. */
+  dispose(): Promise<void>;
+  /** Advisory: is the subscription currently believed to be open? */
+  readonly connected: boolean;
+}
+
+// ─── Noop default ─────────────────────────────────────────────────────────
+
+/**
+ * Default `PushReceiver` used when no real queue is wired (or the flag is OFF
+ * at the factory). `start()` flips `connected` true; `dispose()` flips it
+ * false. No subscription work, no events. Mirrors {@link NoopPushTransport}.
+ */
+export class NoopPushReceiver implements PushReceiver {
+  private _connected = false;
+
+  get connected(): boolean {
+    return this._connected;
+  }
+
+  async start(): Promise<void> {
+    this._connected = true;
+  }
+
+  async dispose(): Promise<void> {
+    this._connected = false;
+  }
+}
+
+// ─── Feature-flag resolution ─────────────────────────────────────────────────
+
+/**
+ * Resolve the enabled boolean from layered config. Precedence (highest first):
+ *   1. `explicit` — `enabled: true|false` wins outright (test contract).
+ *   2. `flagProvider.isEnabled(tenantId)` — injected per-tenant provider.
+ *   3. default {@link EnvTenantListFlagProvider} from `env ?? process.env`.
+ */
+function resolveEnabled(args: {
+  explicit: boolean | undefined;
+  flagProvider: EventDrivenPushFlagProvider | undefined;
+  tenantId: string;
+  env: Record<string, string | undefined> | undefined;
+}): boolean {
+  if (args.explicit !== undefined) return args.explicit;
+  if (args.flagProvider !== undefined) {
+    return args.flagProvider.isEnabled(args.tenantId);
+  }
+  return defaultFlagProvider(args.env ?? process.env).isEnabled(args.tenantId);
+}
+
+// ─── SQS receiver ─────────────────────────────────────────────────────────
+
+/**
+ * Configuration for {@link SqsPushReceiver}.
+ */
+export interface SqsPushReceiverOptions {
+  /**
+   * Tenant id this receiver subscribes to. Each instance binds exactly one
+   * tenant — cross-tenant isolation is enforced by the subscription boundary
+   * (this queue belongs to this tenant), not post-hoc filtering. (US-010)
+   */
+  tenantId: string;
+  /**
+   * The caller's own per-tenant SQS queue URL (minted server-side by the
+   * provisioning Lambda and subscribed to `sync-push-{tenantId}`). The
+   * receiver polls ONLY this URL.
+   */
+  queueUrl: string;
+  /** The injected SQS client. Tests pass a fake; production an SDK adapter. */
+  sqs: SqsClientLike;
+  /**
+   * The sync engine that performs the actual targeted pull. The receiver only
+   * invokes this; errors are logged + isolated from the loop.
+   */
+  syncFn: SyncEngineFn;
+  /** Structured logger. Default: a no-op (quiet). */
+  logger?: ReceiverLogger;
+  /**
+   * Feature-flag override — wins over provider + env when set explicitly.
+   * When the resolved value is false, `start()` is a no-op, `connected` stays
+   * false, and NO queue is polled. (AC#4)
+   */
+  enabled?: boolean;
+  /** Per-tenant flag provider (US-008). Consulted when `enabled` is unset. */
+  flagProvider?: EventDrivenPushFlagProvider;
+  /** Env snapshot for flag resolution. Default: `process.env`. */
+  env?: Record<string, string | undefined>;
+  /** SQS long-poll wait seconds. Default {@link DEFAULT_WAIT_TIME_SECONDS}. */
+  waitTimeSeconds?: number;
+  /** Max messages per receive. Default {@link DEFAULT_MAX_MESSAGES}. */
+  maxMessages?: number;
+  /** Max time `dispose()` waits for an in-flight syncFn after abort. */
+  disposeDrainMs?: number;
+  /** Reconnect backoff config. */
+  reconnect?: {
+    initialMs?: number;
+    maxMs?: number;
+    jitter?: boolean;
+  };
+  /**
+   * Sleep seam for backoff (tests inject a fast/abortable sleep). Default:
+   * host `setTimeout` that resolves early on abort.
+   */
+  sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+  /**
+   * Best-effort latency-metric publish (US-011). Called on the receive-success
+   * path with the measured save→visible latency. Default:
+   * {@link publishSyncLatencyMetric}; tests inject a spy. (AC#1/#3)
+   */
+  publishMetric?: PublishMetricFn;
+  /**
+   * Clock for latency measurement + metric timestamps. Default
+   * `() => Date.now()`. Tests inject a fake clock to assert the latency value.
+   */
+  now?: () => number;
+}
+
+/**
+ * Real client `PushReceiver` backed by a per-tenant SQS queue.
+ *
+ * Poll loop: long-poll `receiveMessage` → for each message, decode + dedupe +
+ * dispatch through `syncFn`, then `deleteMessage` on successful handoff. A
+ * `receiveMessage` rejection is treated as a transient disconnect: log, back
+ * off, resume. SQS retention covers offline catch-up; dedupe covers redelivery.
+ */
+export class SqsPushReceiver implements PushReceiver {
+  private readonly tenantId: string;
+  private readonly queueUrl: string;
+  private readonly sqs: SqsClientLike;
+  private readonly syncFn: SyncEngineFn;
+  private readonly logger: ReceiverLogger;
+  private readonly enabled: boolean;
+  private readonly waitTimeSeconds: number;
+  private readonly maxMessages: number;
+  private readonly disposeDrainMs: number;
+  private readonly reconnectInitialMs: number;
+  private readonly reconnectMaxMs: number;
+  private readonly reconnectJitter: boolean;
+  private readonly sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+  private readonly publishMetric: PublishMetricFn;
+  private readonly now: () => number;
+
+  private _connected = false;
+  private disposed = false;
+  private disposing = false;
+  private disposePromise: Promise<void> | null = null;
+
+  /** Abort signal shared by the poll loop + in-flight sync; fired on dispose. */
+  private loopAbort: AbortController | null = null;
+  /** The running poll loop promise; awaited (best-effort) during dispose. */
+  private loopPromise: Promise<void> | null = null;
+  /** AbortController for the in-flight syncFn; refreshed each dispatch. */
+  private inFlightAbort: AbortController | null = null;
+  private inFlightSync: Promise<void> | null = null;
+
+  /** Per-path highest sequence number already PROCESSED by syncFn. */
+  private readonly seenSequencePerPath = new Map<string, number>();
+
+  private _processedCount = 0;
+  private _dedupedCount = 0;
+  private _decodeFailureCount = 0;
+  private _receiveErrorCount = 0;
+
+  constructor(opts: SqsPushReceiverOptions) {
+    if (!opts.tenantId || opts.tenantId.trim() === "") {
+      throw new Error("SqsPushReceiver: tenantId is required");
+    }
+    if (!opts.queueUrl || opts.queueUrl.trim() === "") {
+      throw new Error("SqsPushReceiver: queueUrl is required");
+    }
+    this.tenantId = opts.tenantId;
+    this.queueUrl = opts.queueUrl;
+    this.sqs = opts.sqs;
+    this.syncFn = opts.syncFn;
+    this.logger = opts.logger ?? NOOP_LOGGER;
+    this.enabled = resolveEnabled({
+      explicit: opts.enabled,
+      flagProvider: opts.flagProvider,
+      tenantId: opts.tenantId,
+      env: opts.env,
+    });
+    this.waitTimeSeconds = opts.waitTimeSeconds ?? DEFAULT_WAIT_TIME_SECONDS;
+    this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
+    this.disposeDrainMs =
+      opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.reconnectInitialMs =
+      opts.reconnect?.initialMs ?? DEFAULT_RECONNECT_INITIAL_MS;
+    this.reconnectMaxMs = opts.reconnect?.maxMs ?? DEFAULT_RECONNECT_MAX_MS;
+    this.reconnectJitter = opts.reconnect?.jitter ?? true;
+    this.sleep = opts.sleep ?? defaultSleep;
+    this.publishMetric =
+      opts.publishMetric ?? ((m) => publishSyncLatencyMetric(m, { logger: undefined }));
+    this.now = opts.now ?? (() => Date.now());
+  }
+
+  // ─── PushReceiver surface ────────────────────────────────────────────────
+
+  get connected(): boolean {
+    return this._connected;
+  }
+
+  async start(): Promise<void> {
+    if (this.disposed) return;
+    if (!this.enabled) {
+      this.logger.info(
+        {
+          event: "receiver.start.disabled",
+          tenantId: this.tenantId,
+          reason: "feature flag off",
+        },
+        "push receiver disabled by feature flag",
+      );
+      return;
+    }
+    if (this.loopAbort !== null) {
+      // Double-start is a no-op — matches PushTransport idempotency posture.
+      this.logger.debug(
+        { event: "receiver.start.noop", tenantId: this.tenantId },
+        "push receiver already started",
+      );
+      return;
+    }
+
+    this.loopAbort = new AbortController();
+    this._connected = true;
+    this.logger.info(
+      { event: "receiver.start", tenantId: this.tenantId, queueUrl: this.queueUrl },
+      "push receiver subscribed (sqs long-poll)",
+    );
+    // Kick the loop off; do NOT await — start() returns once subscribed.
+    this.loopPromise = this.pollLoop(this.loopAbort.signal);
+  }
+
+  async dispose(): Promise<void> {
+    if (this.disposed) return;
+    if (this.disposePromise !== null) return this.disposePromise;
+    this.disposing = true;
+
+    this.disposePromise = (async () => {
+      // Stop the poll loop + abort any in-flight long-poll / syncFn.
+      try {
+        this.loopAbort?.abort();
+      } catch {
+        /* AbortController.abort never throws on Node; defensive. */
+      }
+      try {
+        this.inFlightAbort?.abort();
+      } catch {
+        /* defensive */
+      }
+
+      if (this.inFlightSync !== null) {
+        const drainDeadline = new Promise<void>((resolve) => {
+          const t = setTimeout(resolve, this.disposeDrainMs);
+          // Unref so a hung syncFn can't keep the loop alive past process exit.
+          (t as { unref?: () => void }).unref?.();
+        });
+        await Promise.race([
+          this.inFlightSync.catch(() => undefined),
+          drainDeadline,
+        ]);
+      }
+
+      // Best-effort: let the poll loop observe the abort and exit.
+      if (this.loopPromise !== null) {
+        await this.loopPromise.catch(() => undefined);
+      }
+
+      this._connected = false;
+      this.disposed = true;
+      this.logger.info(
+        {
+          event: "receiver.stop",
+          tenantId: this.tenantId,
+          processed: this._processedCount,
+          deduped: this._dedupedCount,
+        },
+        "push receiver stopped",
+      );
+    })();
+
+    return this.disposePromise;
+  }
+
+  // ─── Observability ─────────────────────────────────────────────────────────
+
+  /** Events that passed dedupe AND completed `syncFn` successfully. */
+  get processedCount(): number {
+    return this._processedCount;
+  }
+  /** Events skipped by dedupe. */
+  get dedupedCount(): number {
+    return this._dedupedCount;
+  }
+  /** Events dropped at the wire-boundary decode step. */
+  get decodeFailureCount(): number {
+    return this._decodeFailureCount;
+  }
+  /** `receiveMessage` failures (transient disconnects) the loop recovered from. */
+  get receiveErrorCount(): number {
+    return this._receiveErrorCount;
+  }
+
+  // ─── Internals ───────────────────────────────────────────────────────────
+
+  /**
+   * The long-poll loop. Runs until the loop abort signal fires (dispose). A
+   * `receiveMessage` rejection is a transient disconnect — log, back off,
+   * resume. Because the SQS queue retains messages, resuming after a blip
+   * replays the gap (catch-up). The loop never throws past this method; it
+   * is fire-and-forgotten by `start()` and awaited best-effort by `dispose()`.
+   */
+  private async pollLoop(signal: AbortSignal): Promise<void> {
+    let attempt = 0;
+    while (!signal.aborted) {
+      let received: { messages: SqsMessageLike[] };
+      try {
+        received = await this.sqs.receiveMessage({
+          queueUrl: this.queueUrl,
+          maxMessages: this.maxMessages,
+          waitTimeSeconds: this.waitTimeSeconds,
+          signal,
+        });
+        attempt = 0; // success → reset backoff
+        if (received.messages.length > 0) {
+          this._connected = true;
+        }
+      } catch (err) {
+        if (signal.aborted) return; // dispose-driven abort — clean exit
+        this._receiveErrorCount += 1;
+        this._connected = false;
+        const backoff = this.computeBackoff(attempt);
+        attempt += 1;
+        this.logger.warn(
+          {
+            event: "receiver.receive.failed",
+            tenantId: this.tenantId,
+            attempt,
+            backoffMs: backoff,
+            err: { message: (err as Error)?.message },
+          },
+          "push receiver receiveMessage failed; backing off (catch-up replay on resume)",
+        );
+        await this.sleep(backoff, signal);
+        continue;
+      }
+
+      for (const msg of received.messages) {
+        if (signal.aborted) return;
+        await this.handleMessage(msg);
+      }
+    }
+  }
+
+  /**
+   * Decode → dedupe → dispatch → delete a single SQS message. Decode failures
+   * and syncFn throws are logged + absorbed (never crash the loop). The message
+   * is deleted only after a successful handoff so an unprocessed message stays
+   * on the queue for redelivery (the dedupe path makes redelivery idempotent).
+   */
+  private async handleMessage(msg: SqsMessageLike): Promise<void> {
+    if (this.disposing || this.disposed) return;
+
+    let validated: PushEvent;
+    try {
+      validated = decodePushEvent(msg.Body ?? "");
+    } catch (err) {
+      this._decodeFailureCount += 1;
+      this.logger.warn(
+        {
+          event: "receiver.decode.failed",
+          tenantId: this.tenantId,
+          messageId: msg.MessageId,
+          err: { message: (err as Error).message },
+        },
+        "push receiver dropped event: decode failed",
+      );
+      // A poison message we can't decode is deleted so it doesn't redeliver
+      // forever — it carries no actionable path. (Defense in depth.)
+      await this.safeDelete(msg);
+      return;
+    }
+
+    const handled = await this.dispatch(validated);
+    if (handled) {
+      // Delete only after the handoff (success OR dedupe-skip — both mean we
+      // don't need this message again). A syncFn throw still counts as handled
+      // for delete purposes: the seen-counter advanced, so a redelivery would
+      // dedupe; the poll/cadence safety net is the recovery path for the throw.
+      await this.safeDelete(msg);
+    }
+  }
+
+  /**
+   * Dedupe + invoke `syncFn`. Returns true once the event has been accounted
+   * for (deduped, or syncFn settled) so the caller can delete the message.
+   * Stores the in-flight promise so `dispose()` can drain it.
+   */
+  private async dispatch(event: PushEvent): Promise<boolean> {
+    if (this.disposing || this.disposed) return false;
+
+    const seen = this.seenSequencePerPath.get(event.relativePath);
+    if (seen !== undefined && event.sequenceNumber <= seen) {
+      this._dedupedCount += 1;
+      this.logger.debug(
+        {
+          event: "receiver.event.deduped",
+          tenantId: this.tenantId,
+          relativePath: event.relativePath,
+          sequenceNumber: event.sequenceNumber,
+          seen,
+        },
+        "push receiver deduped event",
+      );
+      return true;
+    }
+
+    // Record BEFORE invoking syncFn — a back-to-back event for the same path
+    // must see this sequence in its dedupe check. The trade-off: a syncFn that
+    // throws still advances the counter ("latest we KNOW about"); the safety
+    // net poll is the recovery path for the throw.
+    this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
+
+    const controller = new AbortController();
+    this.inFlightAbort = controller;
+    const ctx: PushReceiverContext = { event, signal: controller.signal };
+
+    const holder: { p: Promise<void> | null } = { p: null };
+    const startMs = this.now();
+    const p: Promise<void> = (async () => {
+      try {
+        this.logger.debug(
+          {
+            event: "receiver.sync.start",
+            tenantId: this.tenantId,
+            relativePath: event.relativePath,
+            sequenceNumber: event.sequenceNumber,
+          },
+          "push receiver invoking sync engine",
+        );
+        await this.syncFn(ctx);
+        this._processedCount += 1;
+        this.logger.debug(
+          {
+            event: "receiver.sync.completed",
+            tenantId: this.tenantId,
+            relativePath: event.relativePath,
+            sequenceNumber: event.sequenceNumber,
+          },
+          "push receiver sync completed",
+        );
+        // ── US-011: 3-log chain (3rd link) + p95 latency metric ──────────
+        // Latency = save-on-A (event.eventTimestamp) → visible-on-B (now),
+        // falling back to the local syncFn duration if the event timestamp is
+        // unparseable. Emitted ONLY on the success path so failed syncs don't
+        // skew p95 toward infinity.
+        const endMs = this.now();
+        const savedAtMs = Date.parse(event.eventTimestamp);
+        const latencyMs = Number.isFinite(savedAtMs)
+          ? Math.max(0, endMs - savedAtMs)
+          : Math.max(0, endMs - startMs);
+        const latencySeconds = latencyMs / 1000;
+        // The 3rd correlated log line — shares `sequenceNumber` with
+        // watcher.emit (client push) and push.receive (server).
+        this.logger.info(
+          {
+            event: "fanout.receive",
+            tenantId: this.tenantId,
+            relativePath: event.relativePath,
+            sequenceNumber: event.sequenceNumber,
+            latencySeconds,
+          },
+          "push receiver fanout-receive (round-trip complete)",
+        );
+        // Best-effort metric. Fire-and-forget so observability never sits on
+        // the dispatch critical path (which gates message deletion) — a slow or
+        // hung CloudWatch call must not delay the delete/next-poll. Errors are
+        // swallowed; publishMetric is itself best-effort.
+        void this.emitLatencyMetric({
+          tenantId: this.tenantId,
+          relativePath: event.relativePath,
+          sequenceNumber: event.sequenceNumber,
+          latencySeconds,
+          timestamp: new Date(endMs),
+        });
+      } catch (err) {
+        // Critical: catch, log, return. A misbehaving sync engine must never
+        // crash the receiver loop. The safety-net poll handles eventual
+        // consistency for failed pulls.
+        const e = err as NodeJS.ErrnoException;
+        this.logger.error(
+          {
+            event: "receiver.sync.failed",
+            tenantId: this.tenantId,
+            relativePath: event.relativePath,
+            sequenceNumber: event.sequenceNumber,
+            err: { message: e?.message, code: e?.code },
+          },
+          "push receiver sync engine threw",
+        );
+      } finally {
+        if (this.inFlightAbort === controller) this.inFlightAbort = null;
+        if (this.inFlightSync === holder.p) this.inFlightSync = null;
+      }
+    })();
+    holder.p = p;
+    this.inFlightSync = p;
+    await p;
+    return true;
+  }
+
+  /** Delete a message, swallowing transport errors (redelivery is harmless). */
+  private async safeDelete(msg: SqsMessageLike): Promise<void> {
+    if (!msg.ReceiptHandle) return;
+    try {
+      await this.sqs.deleteMessage({
+        queueUrl: this.queueUrl,
+        receiptHandle: msg.ReceiptHandle,
+      });
+    } catch (err) {
+      this.logger.warn(
+        {
+          event: "receiver.delete.failed",
+          tenantId: this.tenantId,
+          messageId: msg.MessageId,
+          err: { message: (err as Error)?.message },
+        },
+        "push receiver failed to delete message (will redeliver; dedupe absorbs)",
+      );
+    }
+  }
+
+  /**
+   * Publish one best-effort latency datum (US-011). Awaits `publishMetric` and
+   * swallows any rejection so a metric-backend outage can never reach the poll
+   * loop. Called fire-and-forget (`void`) off the dispatch critical path.
+   */
+  private async emitLatencyMetric(metric: SyncLatencyMetric): Promise<void> {
+    try {
+      await this.publishMetric(metric);
+    } catch (metricErr) {
+      this.logger.warn(
+        {
+          event: "receiver.metric.failed",
+          tenantId: metric.tenantId,
+          sequenceNumber: metric.sequenceNumber,
+          err: { message: (metricErr as Error)?.message },
+        },
+        "push receiver failed to publish latency metric (ignored)",
+      );
+    }
+  }
+
+  /** Exponential backoff (capped) with optional full-jitter. */
+  private computeBackoff(attempt: number): number {
+    const exp = Math.min(
+      this.reconnectMaxMs,
+      this.reconnectInitialMs * 2 ** attempt,
+    );
+    if (!this.reconnectJitter) return exp;
+    return Math.floor(Math.random() * exp);
+  }
+}
+
+// ─── In-memory receiver (unit-test transport analogue) ───────────────────────
+
+/**
+ * A tiny in-process fanout the {@link InMemoryPushReceiver} subscribes against.
+ * Models SNS publish + the per-client SQS queue's disconnect buffering so unit
+ * tests can drive the receive path without AWS. `publish` raw strings (the
+ * wire form) so decode-failure paths are testable too.
+ */
+export class InMemoryFanout {
+  private readonly subscribers = new Set<(raw: string) => void>();
+
+  subscribe(handler: (raw: string) => void): () => void {
+    this.subscribers.add(handler);
+    return () => this.subscribers.delete(handler);
+  }
+
+  /** Publish a raw (already-encoded) message body to all subscribers. */
+  publish(raw: string): void {
+    for (const h of [...this.subscribers]) h(raw);
+  }
+}
+
+/** Options for {@link InMemoryPushReceiver}. */
+export interface InMemoryPushReceiverOptions {
+  tenantId: string;
+  fanout: InMemoryFanout;
+  syncFn: SyncEngineFn;
+  logger?: ReceiverLogger;
+  enabled?: boolean;
+  flagProvider?: EventDrivenPushFlagProvider;
+  env?: Record<string, string | undefined>;
+  disposeDrainMs?: number;
+}
+
+/**
+ * In-memory receiver paired with {@link InMemoryFanout}. Powers the unit
+ * tests for dedupe, reconnect-replay, flag gating, and dispose-drain WITHOUT
+ * any AWS SDK. The dedupe / dispatch / dispose semantics are identical to
+ * {@link SqsPushReceiver} (shared design); the disconnect buffer is the
+ * in-process analogue of the per-client SQS queue's 14-day retention.
+ */
+export class InMemoryPushReceiver implements PushReceiver {
+  private readonly tenantId: string;
+  private readonly fanout: InMemoryFanout;
+  private readonly syncFn: SyncEngineFn;
+  private readonly logger: ReceiverLogger;
+  private readonly enabled: boolean;
+  private readonly disposeDrainMs: number;
+
+  private _connected = false;
+  private disposed = false;
+  private disposing = false;
+  private disposePromise: Promise<void> | null = null;
+  private unsubscribe: (() => void) | null = null;
+
+  private disconnectedFlag = false;
+  private readonly pendingDuringDisconnect: PushEvent[] = [];
+  private readonly seenSequencePerPath = new Map<string, number>();
+
+  private inFlightAbort: AbortController | null = null;
+  private inFlightSync: Promise<void> | null = null;
+
+  private _processedCount = 0;
+  private _dedupedCount = 0;
+  private _decodeFailureCount = 0;
+
+  constructor(opts: InMemoryPushReceiverOptions) {
+    this.tenantId = opts.tenantId;
+    this.fanout = opts.fanout;
+    this.syncFn = opts.syncFn;
+    this.logger = opts.logger ?? NOOP_LOGGER;
+    this.disposeDrainMs =
+      opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.enabled = resolveEnabled({
+      explicit: opts.enabled,
+      flagProvider: opts.flagProvider,
+      tenantId: opts.tenantId,
+      env: opts.env,
+    });
+  }
+
+  get connected(): boolean {
+    return this._connected;
+  }
+
+  async start(): Promise<void> {
+    if (this.disposed) return;
+    if (!this.enabled) {
+      this.logger.info(
+        { event: "receiver.start.disabled", tenantId: this.tenantId },
+        "push receiver disabled by feature flag",
+      );
+      return;
+    }
+    if (this.unsubscribe !== null) return; // double-start no-op
+
+    this.unsubscribe = this.fanout.subscribe((raw) => {
+      let validated: PushEvent;
+      try {
+        validated = decodePushEvent(raw);
+      } catch (err) {
+        this._decodeFailureCount += 1;
+        this.logger.warn(
+          {
+            event: "receiver.decode.failed",
+            tenantId: this.tenantId,
+            err: { message: (err as Error).message },
+          },
+          "push receiver dropped event: decode failed",
+        );
+        return;
+      }
+      if (this.disconnectedFlag) {
+        this.pendingDuringDisconnect.push(validated);
+        return;
+      }
+      void this.dispatch(validated);
+    });
+    this._connected = true;
+    this.logger.info(
+      { event: "receiver.start", tenantId: this.tenantId },
+      "push receiver subscribed (in-memory)",
+    );
+  }
+
+  async dispose(): Promise<void> {
+    if (this.disposed) return;
+    if (this.disposePromise !== null) return this.disposePromise;
+    this.disposing = true;
+    this.disposePromise = (async () => {
+      try {
+        this.inFlightAbort?.abort();
+      } catch {
+        /* defensive */
+      }
+      if (this.inFlightSync !== null) {
+        const drainDeadline = new Promise<void>((resolve) => {
+          const t = setTimeout(resolve, this.disposeDrainMs);
+          (t as { unref?: () => void }).unref?.();
+        });
+        await Promise.race([
+          this.inFlightSync.catch(() => undefined),
+          drainDeadline,
+        ]);
+      }
+      if (this.unsubscribe !== null) {
+        try {
+          this.unsubscribe();
+        } catch {
+          /* defensive */
+        }
+        this.unsubscribe = null;
+      }
+      this._connected = false;
+      this.disposed = true;
+      this.logger.info(
+        {
+          event: "receiver.stop",
+          tenantId: this.tenantId,
+          processed: this._processedCount,
+          deduped: this._dedupedCount,
+        },
+        "push receiver stopped",
+      );
+    })();
+    return this.disposePromise;
+  }
+
+  // ─── Test hooks (model the SQS retention buffer) ────────────────────────────
+
+  get processedCount(): number {
+    return this._processedCount;
+  }
+  get dedupedCount(): number {
+    return this._dedupedCount;
+  }
+  get decodeFailureCount(): number {
+    return this._decodeFailureCount;
+  }
+  get bufferedCount(): number {
+    return this.pendingDuringDisconnect.length;
+  }
+
+  /** Emulate a network blip — events buffer instead of dispatching. */
+  simulateDisconnect(): void {
+    if (!this.enabled || this.unsubscribe === null) return;
+    this.disconnectedFlag = true;
+    this._connected = false;
+  }
+
+  /** Emulate reconnect — drain the buffer through the same dedupe path. */
+  simulateReconnect(): void {
+    if (!this.disconnectedFlag) return;
+    this.disconnectedFlag = false;
+    this._connected = true;
+    const queued = this.pendingDuringDisconnect.splice(0);
+    for (const evt of queued) void this.dispatch(evt);
+  }
+
+  private dispatch(event: PushEvent): Promise<void> {
+    if (this.disposing || this.disposed) return Promise.resolve();
+
+    const seen = this.seenSequencePerPath.get(event.relativePath);
+    if (seen !== undefined && event.sequenceNumber <= seen) {
+      this._dedupedCount += 1;
+      return Promise.resolve();
+    }
+    this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
+
+    const controller = new AbortController();
+    this.inFlightAbort = controller;
+    const ctx: PushReceiverContext = { event, signal: controller.signal };
+    const holder: { p: Promise<void> | null } = { p: null };
+    const p: Promise<void> = (async () => {
+      try {
+        await this.syncFn(ctx);
+        this._processedCount += 1;
+      } catch (err) {
+        this.logger.error(
+          {
+            event: "receiver.sync.failed",
+            tenantId: this.tenantId,
+            relativePath: event.relativePath,
+            err: { message: (err as Error)?.message },
+          },
+          "push receiver sync engine threw",
+        );
+      } finally {
+        if (this.inFlightAbort === controller) this.inFlightAbort = null;
+        if (this.inFlightSync === holder.p) this.inFlightSync = null;
+      }
+    })();
+    holder.p = p;
+    this.inFlightSync = p;
+    return p;
+  }
+}
+
+// ─── Factory ───────────────────────────────────────────────────────────────
+
+/**
+ * Build a PushReceiver. The daemon defaults to the noop so wiring is
+ * regression-safe — production deployments wire the SQS impl explicitly once
+ * the server provisioning Lambda mints a queue URL. Mirrors PushTransport's
+ * noop-default opt-in posture.
+ */
+export type CreatePushReceiverOptions =
+  | (SqsPushReceiverOptions & { kind?: "sqs" })
+  | { kind: "noop" };
+
+export function createPushReceiver(
+  opts: CreatePushReceiverOptions,
+): PushReceiver {
+  if ("kind" in opts && opts.kind === "noop") {
+    return new NoopPushReceiver();
+  }
+  if ("queueUrl" in opts && "sqs" in opts) {
+    return new SqsPushReceiver(opts);
+  }
+  return new NoopPushReceiver();
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────────────
+
+/**
+ * Default sleep that resolves after `ms` OR promptly on abort (so a dispose
+ * during a backoff wait doesn't block). Never rejects.
+ */
+function defaultSleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise<void>((resolve) => {
+    if (signal.aborted) return resolve();
+    const t = setTimeout(() => {
+      signal.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    (t as { unref?: () => void }).unref?.();
+    const onAbort = (): void => {
+      clearTimeout(t);
+      resolve();
+    };
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+}