@indigoai-us/hq-cloud 6.3.0 → 6.3.2

package/src/sync/event-sync.ts

@@ -0,0 +1,481 @@
+/**
+ * Event-driven sync wiring — Phase 3 of event-driven-sync-menubar
+ * (US-017 publish / US-018 receive / US-019 rollout gate).
+ *
+ * Phases 1–2 built every piece but left them unconnected: the watcher runs
+ * targeted *push passes* (S3 upload) but never publishes a PushEvent, and the
+ * runner's receiver seam defaults to {@link NoopPushReceiver}. This module is
+ * the connective tissue that turns both on for enrolled accounts:
+ *
+ *  - {@link resolveEventSync} — the rollout gate. EXACT-email allowlist
+ *    (mirrors `resolvePresignTransport`'s shape in sync-runner.ts, but full
+ *    address, not domain suffix) + `HQ_SYNC_EVENT_SYNC` env override in both
+ *    directions. ONE gate governs publish AND receive so a device is never
+ *    publish-only or receive-only in a half-rolled state.
+ *  - {@link subscribeSyncReceive} — `POST /v1/sync/subscribe` (US-015/US-016):
+ *    mints the per-device queue and returns `{queueUrl, region, credentials}`,
+ *    where `credentials` are short-lived STS creds scoped to receive/delete on
+ *    exactly that queue.
+ *  - {@link sqsClientFromAwsSdk} — the doc-promised thin adapter from the AWS
+ *    SDK `SQSClient` to the receiver's narrow {@link SqsClientLike} seam.
+ *  - {@link createRefreshingSqsClient} — wraps the adapter with credential
+ *    lifecycle: proactive re-vend before expiry (skew window) + one reactive
+ *    retry on an expiry-class error. The queue URL is stable across re-vends
+ *    (same device → same queue, idempotent endpoint); only creds rotate.
+ *  - {@link startEventSync} — the wiring entry the runner calls from the
+ *    `--event-push` watch block when the gate is ON. Resolves tenant + device
+ *    identity, builds the {@link HttpPushTransport} + {@link PushEventEmitter}
+ *    (publish leg) and the {@link SqsPushReceiver} (receive leg, self-echo
+ *    filtered), and returns handles. Any startup failure degrades to
+ *    poll-only — it NEVER takes the daemon down.
+ *
+ * The 10-minute `--poll-remote-ms` pass remains the correctness backstop for
+ * every path here; event delivery is best-effort by design.
+ */
+
+import {
+  SQSClient,
+  ReceiveMessageCommand,
+  DeleteMessageCommand,
+} from "@aws-sdk/client-sqs";
+
+import { HttpPushTransport, type AuthTokenSource } from "./push-transport.js";
+import {
+  SqsPushReceiver,
+  type PushReceiver,
+  type SqsClientLike,
+  type SyncEngineFn,
+} from "./push-receiver.js";
+import { PushEventEmitter } from "../watcher.js";
+import type { TreeChangeBatch } from "../watcher.js";
+
+// ─── US-019: rollout gate ───────────────────────────────────────────────────
+
+/**
+ * Accounts enrolled in event-driven sync (publish + receive).
+ *
+ * EXACT full-address matching, case-insensitive — NOT a domain suffix. The
+ * single-account Phase 3 rollout (2026-06-10) targets the operator's own
+ * devices; `xhassaan@getindigo.ai` and `hassaan@getindigo.ai.evil.com` must
+ * never match. Broadening later is an entry here (or a domain-set like
+ * `PRESIGN_ROLLOUT_DOMAINS` once GA'd).
+ */
+export const EVENT_SYNC_ROLLOUT_EMAILS: ReadonlySet<string> = new Set([
+  "hassaan@getindigo.ai",
+]);
+
+/**
+ * Decide whether this session runs event-driven sync (publish + receive).
+ *
+ * Mirrors `resolvePresignTransport` precedence: `HQ_SYNC_EVENT_SYNC`
+ * overrides in both directions (`1`/`true`/`yes`/`on` → force on,
+ * `0`/`false`/`no`/`off` → force off) so unenrolled testers can exercise it
+ * and enrolled accounts can be rolled back without a release. An unset/blank
+ * override falls through to the exact-email check; an unrecognized override
+ * value is ignored (email check wins).
+ */
+export function resolveEventSync(
+  email: string | undefined,
+  override: string | undefined,
+): boolean {
+  const o = (override ?? "").trim().toLowerCase();
+  if (o === "1" || o === "true" || o === "yes" || o === "on") return true;
+  if (o === "0" || o === "false" || o === "no" || o === "off") return false;
+  if (typeof email !== "string") return false;
+  return EVENT_SYNC_ROLLOUT_EMAILS.has(email.trim().toLowerCase());
+}
+
+// ─── US-018: subscribe + credentials ────────────────────────────────────────
+
+export interface SubscribeSyncCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken: string;
+  /** ISO8601 expiry of the vended STS credentials. */
+  expiration: string;
+}
+
+export interface SubscribeSyncResponse {
+  /** The caller's own per-device queue URL (stable across calls). */
+  queueUrl: string;
+  /** Region the queue lives in. */
+  region: string;
+  /** Short-lived creds scoped to receive/delete on exactly this queue. */
+  credentials: SubscribeSyncCredentials;
+}
+
+/** Minimal fetch seam (matches push-transport.ts's FetchLike posture). */
+type FetchLike = (
+  url: string,
+  init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+    signal?: AbortSignal;
+  },
+) => Promise<{ ok: boolean; status: number; text(): Promise<string> }>;
+
+function asNonEmptyString(v: unknown, field: string): string {
+  if (typeof v !== "string" || v.length === 0) {
+    throw new Error(`subscribeSyncReceive: response missing ${field}`);
+  }
+  return v;
+}
+
+/**
+ * `POST /v1/sync/subscribe` — provision (idempotently) this device's queue
+ * and vend fresh receive credentials. Auth mirrors HttpPushTransport: Bearer
+ * token resolved per-call via the supplied source.
+ */
+export async function subscribeSyncReceive(opts: {
+  apiUrl: string;
+  authToken: AuthTokenSource;
+  deviceId: string;
+  timeoutMs?: number;
+  fetchImpl?: FetchLike;
+}): Promise<SubscribeSyncResponse> {
+  const apiUrl = opts.apiUrl.replace(/\/+$/, "");
+  const tok = opts.authToken;
+  const token = typeof tok === "function" ? await tok() : tok;
+  const fetchImpl =
+    opts.fetchImpl ??
+    ((input, init) => (globalThis.fetch as unknown as FetchLike)(input, init));
+
+  const controller = new AbortController();
+  const timeout = setTimeout(
+    () => controller.abort(),
+    opts.timeoutMs ?? 15_000,
+  );
+  let res: Awaited<ReturnType<FetchLike>>;
+  try {
+    res = await fetchImpl(`${apiUrl}/v1/sync/subscribe`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ deviceId: opts.deviceId }),
+      signal: controller.signal,
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    throw new Error(
+      `subscribeSyncReceive: POST /v1/sync/subscribe failed (${res.status}): ${text.slice(0, 300)}`,
+    );
+  }
+  const parsed: unknown = JSON.parse(await res.text());
+  const obj = parsed as Record<string, unknown>;
+  const creds = (obj.credentials ?? {}) as Record<string, unknown>;
+  return {
+    queueUrl: asNonEmptyString(obj.queueUrl, "queueUrl"),
+    region: asNonEmptyString(obj.region, "region"),
+    credentials: {
+      accessKeyId: asNonEmptyString(creds.accessKeyId, "credentials.accessKeyId"),
+      secretAccessKey: asNonEmptyString(
+        creds.secretAccessKey,
+        "credentials.secretAccessKey",
+      ),
+      sessionToken: asNonEmptyString(
+        creds.sessionToken,
+        "credentials.sessionToken",
+      ),
+      expiration: asNonEmptyString(creds.expiration, "credentials.expiration"),
+    },
+  };
+}
+
+// ─── SQS SDK adapter ────────────────────────────────────────────────────────
+
+/**
+ * Adapt the AWS SDK `SQSClient` to the receiver's narrow {@link SqsClientLike}
+ * seam (the doc-promised `sqsClientFromAwsSdk` from push-receiver.ts). The
+ * abort signal is forwarded so `dispose()` can cut a 20s long-poll short.
+ */
+export function sqsClientFromAwsSdk(
+  client: Pick<SQSClient, "send">,
+): SqsClientLike {
+  return {
+    async receiveMessage({ queueUrl, maxMessages, waitTimeSeconds, signal }) {
+      const out = await client.send(
+        new ReceiveMessageCommand({
+          QueueUrl: queueUrl,
+          MaxNumberOfMessages: maxMessages,
+          WaitTimeSeconds: waitTimeSeconds,
+        }),
+        { abortSignal: signal },
+      );
+      return {
+        messages: (out.Messages ?? []).map((m) => ({
+          Body: m.Body,
+          ReceiptHandle: m.ReceiptHandle,
+          MessageId: m.MessageId,
+        })),
+      };
+    },
+    async deleteMessage({ queueUrl, receiptHandle }) {
+      await client.send(
+        new DeleteMessageCommand({
+          QueueUrl: queueUrl,
+          ReceiptHandle: receiptHandle,
+        }),
+      );
+    },
+  };
+}
+
+// ─── Refreshing SQS client (credential lifecycle) ───────────────────────────
+
+/**
+ * Error names the SQS SDK surfaces when STS session creds have expired or
+ * become invalid. Used for the reactive retry-once path; the proactive
+ * skew-window refresh below should make these rare.
+ */
+const EXPIRY_ERROR_NAMES = new Set([
+  "ExpiredToken",
+  "ExpiredTokenException",
+  "InvalidSecurityToken",
+  "InvalidClientTokenId",
+  "UnrecognizedClientException",
+]);
+
+function isExpiryError(err: unknown): boolean {
+  return err instanceof Error && EXPIRY_ERROR_NAMES.has(err.name);
+}
+
+/** Re-vend this long before the recorded expiry (proactive refresh window). */
+const REFRESH_SKEW_MS = 120_000;
+
+export interface RefreshingSqsClientOptions {
+  /** The initial subscribe response (creds + region + queue URL). */
+  initial: SubscribeSyncResponse;
+  /** Re-vend: called when creds are near/past expiry. Idempotent server-side. */
+  subscribe: () => Promise<SubscribeSyncResponse>;
+  /**
+   * Build the underlying narrow client from a subscribe response. Default:
+   * AWS SDK `SQSClient` via {@link sqsClientFromAwsSdk}. Tests inject a fake.
+   */
+  buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
+  /** Clock seam (tests). Default `Date.now`. */
+  now?: () => number;
+}
+
+function defaultBuildSqs(resp: SubscribeSyncResponse): SqsClientLike {
+  return sqsClientFromAwsSdk(
+    new SQSClient({
+      region: resp.region,
+      credentials: {
+        accessKeyId: resp.credentials.accessKeyId,
+        secretAccessKey: resp.credentials.secretAccessKey,
+        sessionToken: resp.credentials.sessionToken,
+        expiration: new Date(resp.credentials.expiration),
+      },
+    }),
+  );
+}
+
+/**
+ * An {@link SqsClientLike} that owns the vended-credential lifecycle:
+ *
+ *  - PROACTIVE: before each call, if the recorded expiry is within the skew
+ *    window, re-subscribe (re-vend) and rebuild the inner client first.
+ *  - REACTIVE: if a call still fails with an expiry-class error (clock skew,
+ *    revocation), re-vend once and retry the call once. Anything else — or a
+ *    second failure — propagates to the receiver's own backoff/reconnect
+ *    loop, whose retention-backed redelivery makes the miss recoverable.
+ *
+ * Concurrent refreshes collapse onto one in-flight subscribe promise.
+ */
+export function createRefreshingSqsClient(
+  opts: RefreshingSqsClientOptions,
+): SqsClientLike {
+  const now = opts.now ?? (() => Date.now());
+  const buildSqs = opts.buildSqs ?? defaultBuildSqs;
+
+  let inner = buildSqs(opts.initial);
+  let expiresAtMs = Date.parse(opts.initial.credentials.expiration);
+  let refreshing: Promise<void> | null = null;
+
+  const refresh = (): Promise<void> => {
+    refreshing ??= (async () => {
+      try {
+        const resp = await opts.subscribe();
+        inner = buildSqs(resp);
+        expiresAtMs = Date.parse(resp.credentials.expiration);
+      } finally {
+        refreshing = null;
+      }
+    })();
+    return refreshing;
+  };
+
+  const ensureFresh = async (): Promise<void> => {
+    if (Number.isFinite(expiresAtMs) && now() >= expiresAtMs - REFRESH_SKEW_MS) {
+      await refresh();
+    }
+  };
+
+  const withRetry = async <T>(call: () => Promise<T>): Promise<T> => {
+    await ensureFresh();
+    try {
+      return await call();
+    } catch (err) {
+      if (!isExpiryError(err)) throw err;
+      await refresh();
+      return await call();
+    }
+  };
+
+  return {
+    receiveMessage: (args) => withRetry(() => inner.receiveMessage(args)),
+    deleteMessage: (args) => withRetry(() => inner.deleteMessage(args)),
+  };
+}
+
+// ─── US-017 + US-018: full wiring ───────────────────────────────────────────
+
+/** Structured line logger seam — the runner passes its stderr logger. */
+export type EventSyncLog = (message: string) => void;
+
+export interface StartEventSyncOptions {
+  hqRoot: string;
+  /** Vault API base URL (the runner's DEFAULT_VAULT_API_URL). */
+  apiUrl: string;
+  /** Cognito access-token source (getter for long-running daemons). */
+  authToken: AuthTokenSource;
+  /** This device's stable id (getOrCreateMachineId). */
+  deviceId: string;
+  /**
+   * Resolve the caller's tenant id (canonical person `prs_*` uid). The server
+   * rejects publishes whose `originTenantId` mismatches the JWT principal, so
+   * this MUST be the same identity the JWT resolves to.
+   */
+  resolveTenantId: () => Promise<string>;
+  /**
+   * The already-routed targeted-pull bridge (the runner's receiverSyncFn,
+   * funneled through its runGuarded mutex). Self-echo filtering happens HERE,
+   * before this is invoked.
+   */
+  syncFn: SyncEngineFn;
+  /** Diagnostic logger (one line per lifecycle event). Default: console.error. */
+  log?: EventSyncLog;
+  // ── test seams ──
+  subscribe?: (deviceId: string) => Promise<SubscribeSyncResponse>;
+  buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
+  transport?: HttpPushTransport;
+  now?: () => number;
+}
+
+export interface EventSyncHandles {
+  /**
+   * Publish PushEvents for a settled change batch. Called by the runner
+   * AFTER the targeted push pass succeeds — an event must never announce
+   * bytes that are not in S3 yet. Fire-and-forget: failures are logged by
+   * the emitter's onError and the cadence poll covers the miss.
+   */
+  publishBatch: (batch: TreeChangeBatch) => void;
+  /** The live receiver (already started). */
+  receiver: PushReceiver;
+  /** This device's id — the runner uses it nowhere else, exposed for logs. */
+  ownDeviceId: string;
+  /** Tear down transport + receiver (runner shutdown path). */
+  dispose: () => Promise<void>;
+}
+
+/**
+ * Bring up the publish + receive legs. Returns `null` (poll-only degradation)
+ * on ANY startup failure — subscribe 5xx, tenant resolution failure, etc. —
+ * after logging the reason. The caller treats `null` as "today's behavior".
+ */
+export async function startEventSync(
+  opts: StartEventSyncOptions,
+): Promise<EventSyncHandles | null> {
+  const log = opts.log ?? ((m: string) => console.error(m));
+  try {
+    const tenantId = await opts.resolveTenantId();
+    const deviceId = opts.deviceId;
+
+    // ── publish leg (US-017) ──
+    const transport =
+      opts.transport ??
+      new HttpPushTransport({
+        apiUrl: opts.apiUrl,
+        pushPath: "/v1/sync/push",
+        authToken: opts.authToken,
+      });
+    await transport.start();
+    const emitter = new PushEventEmitter({
+      originTenantId: tenantId,
+      originDeviceId: deviceId,
+      transport,
+      // The rollout gate (resolveEventSync) already said yes — the emitter's
+      // per-tenant env flag is a server-side convention, not the client gate.
+      flagProvider: { isEnabled: () => true },
+      // Wall-clock sequence numbers: monotonic per device ACROSS daemon
+      // restarts (the emitter's default in-process counter restarts at 0,
+      // which would make peers' highest-seq dedupe drop every post-restart
+      // event). Receiver dedupe is per-path highest-seq, so ms-epoch values
+      // also stay comparable when two devices touch the same path.
+      getSequenceNumber: () => (opts.now ?? Date.now)(),
+      onError: (err, ctx) =>
+        log(
+          `event-sync: publish failed for ${ctx.relativePath ?? "?"}: ${err.message}`,
+        ),
+    });
+
+    // ── receive leg (US-018) ──
+    const subscribe =
+      opts.subscribe ??
+      ((dev: string) =>
+        subscribeSyncReceive({
+          apiUrl: opts.apiUrl,
+          authToken: opts.authToken,
+          deviceId: dev,
+        }));
+    const initial = await subscribe(deviceId);
+    const sqs = createRefreshingSqsClient({
+      initial,
+      subscribe: () => subscribe(deviceId),
+      buildSqs: opts.buildSqs,
+      now: opts.now,
+    });
+    const receiver = new SqsPushReceiver({
+      tenantId,
+      queueUrl: initial.queueUrl,
+      sqs,
+      // Self-echo filter: this device's own publishes fan out to its own
+      // queue too — pulling them back is wasted work at best and a conflict
+      // source at worst. Skip before bridging into the sync engine.
+      syncFn: async (ctx) => {
+        if (ctx.event.originDeviceId === deviceId) return;
+        await opts.syncFn(ctx);
+      },
+      // The client rollout gate is the authority; the env-driven per-tenant
+      // flag provider is a server-side convention not set on user machines.
+      enabled: true,
+    });
+    await receiver.start();
+
+    log(
+      `event-sync: live (tenant=${tenantId} device=${deviceId} queue=${initial.queueUrl})`,
+    );
+
+    return {
+      publishBatch: (batch) => void emitter.emitForBatch(batch),
+      receiver,
+      ownDeviceId: deviceId,
+      dispose: async () => {
+        await receiver.dispose().catch(() => undefined);
+        await transport.dispose().catch(() => undefined);
+      },
+    };
+  } catch (err) {
+    log(
+      `event-sync: startup failed, continuing poll-only: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return null;
+  }
+}

package/test/e2e/sync/cross-tenant-isolation.test.ts

@@ -500,3 +500,129 @@ describe("US-010: cross-tenant isolation (BLOCKING — failure is P0)", () => {
     5_000,
   );
 });
+
+// ---------------------------------------------------------------------------
+// Phase 3 extension (US-018) — receive path with VENDED credentials
+// ---------------------------------------------------------------------------
+//
+// US-016 moved queue access from "client holds no way to read" to "subscribe
+// vends per-device STS creds scoped to exactly one queue ARN". The server-
+// side policy-document tests pin the IAM scoping; THIS extension pins the
+// client half of the same invariant: the startEventSync wiring polls ONLY
+// the queue URL its own subscribe returned — a tenant-B (or peer-device)
+// queue is never touched, and a same-tenant peer's events flow while the
+// device's own echoes are filtered.
+
+import {
+  startEventSync,
+  type SubscribeSyncResponse,
+} from "../../../src/sync/event-sync.js";
+import type {
+  SqsClientLike,
+  PushReceiverContext,
+} from "../../../src/sync/push-receiver.js";
+import type { PushEvent } from "../../../src/sync/push-event.js";
+
+describe("US-018: receive path with vended credentials (isolation extension)", () => {
+  function vendedResponse(tenant: string, device: string): SubscribeSyncResponse {
+    return {
+      queueUrl: `https://sqs.us-east-1.amazonaws.com/000000000000/sync-push-${tenant}-${device}`,
+      region: "us-east-1",
+      credentials: {
+        accessKeyId: `AKIA-${tenant}-${device}`,
+        secretAccessKey: "s",
+        sessionToken: "t",
+        expiration: "2099-01-01T00:00:00.000Z",
+      },
+    };
+  }
+
+  function phase3Event(overrides: Partial<PushEvent>): PushEvent {
+    return {
+      relativePath: "companies/indigo/docs/x.md",
+      contentHash: `sha256:${"b".repeat(64)}`,
+      mtime: "2026-06-10T12:00:00.000Z",
+      originDeviceId: "peer",
+      originTenantId: "prs_A",
+      sequenceNumber: 10,
+      eventTimestamp: "2026-06-10T12:00:00.000Z",
+      ...overrides,
+    };
+  }
+
+  it("polls ONLY the queue URL its own subscribe returned, with its own vended creds", async () => {
+    // The suite-level beforeEach arms fake timers for the simulated-60s
+    // tests; this extension drives a REAL async poll loop (vi.waitFor +
+    // receiver microtasks), so switch back. afterEach re-reals anyway.
+    vi.useRealTimers();
+    const polledQueues = new Set<string>();
+    const credsUsed = new Set<string>();
+    let deliver: ((body: string) => void) | undefined;
+
+    const pulled: PushEvent[] = [];
+    const handles = await startEventSync({
+      hqRoot: "/tmp/hq-a1",
+      apiUrl: "https://api.example.com",
+      authToken: "jwt-A1",
+      deviceId: "devA1",
+      resolveTenantId: async () => "prs_A",
+      syncFn: async (ctx: PushReceiverContext) => {
+        pulled.push(ctx.event);
+      },
+      subscribe: async () => vendedResponse("prs_A", "devA1"),
+      buildSqs: (resp): SqsClientLike => {
+        credsUsed.add(resp.credentials.accessKeyId);
+        return {
+          receiveMessage: ({ queueUrl, signal }) =>
+            new Promise((resolve) => {
+              polledQueues.add(queueUrl);
+              signal.addEventListener(
+                "abort",
+                () => resolve({ messages: [] }),
+                { once: true },
+              );
+              deliver = (body: string) => {
+                deliver = undefined;
+                resolve({ messages: [{ Body: body, ReceiptHandle: "rh" }] });
+              };
+            }),
+          deleteMessage: async () => {},
+        };
+      },
+      log: () => {},
+    });
+    expect(handles).not.toBeNull();
+
+    await vi.waitFor(() => {
+      if (!deliver) throw new Error("not polling yet");
+    });
+
+    // The ONLY queue ever polled is this device's own vended queue, with this
+    // device's own vended credentials — the structural mirror of the server
+    // session policy (single queue ARN, nothing else).
+    expect([...polledQueues]).toEqual([
+      "https://sqs.us-east-1.amazonaws.com/000000000000/sync-push-prs_A-devA1",
+    ]);
+    expect([...credsUsed]).toEqual(["AKIA-prs_A-devA1"]);
+
+    // A same-tenant PEER event is bridged into the targeted pull…
+    deliver!(
+      JSON.stringify(phase3Event({ originDeviceId: "devA2", sequenceNumber: 11 })),
+    );
+    await vi.waitFor(() => expect(pulled).toHaveLength(1));
+    expect(pulled[0].originDeviceId).toBe("devA2");
+
+    // …while the device's OWN echo is filtered before the sync engine.
+    await vi.waitFor(() => {
+      if (!deliver) throw new Error("not polling again yet");
+    });
+    deliver!(
+      JSON.stringify(phase3Event({ originDeviceId: "devA1", sequenceNumber: 12 })),
+    );
+    // Give the loop a beat — the echo must NOT add a pull.
+    await new Promise((r) => setTimeout(r, 20));
+    expect(pulled).toHaveLength(1);
+
+    await handles!.dispose();
+  });
+});