@indigoai-us/hq-cloud 6.15.0 → 6.15.2

package/src/sync/event-sync.ts

@@ -1,545 +0,0 @@
-/**
- * Event-driven sync wiring — Phase 3 of event-driven-sync-menubar
- * (US-017 publish / US-018 receive / US-019 rollout gate).
- *
- * Phases 1–2 built every piece but left them unconnected: the watcher runs
- * targeted *push passes* (S3 upload) but never publishes a PushEvent, and the
- * runner's receiver seam defaults to {@link NoopPushReceiver}. This module is
- * the connective tissue that turns both on after a server grant:
- *
- *  - {@link resolveEventSync} — a negative-only local kill switch. A server
- *    grant remains the only positive authority for publish and receive.
- *  - {@link subscribeSyncReceive} — `POST /v1/sync/subscribe` (US-015/US-016):
- *    mints the per-device queue and returns `{queueUrl, region, credentials}`,
- *    where `credentials` are short-lived STS creds scoped to receive/delete on
- *    exactly that queue.
- *  - {@link sqsClientFromAwsSdk} — the doc-promised thin adapter from the AWS
- *    SDK `SQSClient` to the receiver's narrow {@link SqsClientLike} seam.
- *  - {@link createRefreshingSqsClient} — wraps the adapter with credential
- *    lifecycle: proactive re-vend before expiry (skew window) + one reactive
- *    retry on an expiry-class error. The queue URL is stable across re-vends
- *    (same device → same queue, idempotent endpoint); only creds rotate.
- *  - {@link startEventSync} — the wiring entry the runner calls from the
- *    `--event-push` watch block unless locally disabled. Resolves tenant + device
- *    identity, builds the {@link HttpPushTransport} + {@link PushEventEmitter}
- *    (publish leg) and the {@link SqsPushReceiver} (receive leg, self-echo
- *    filtered), and returns handles. Any startup failure degrades to
- *    poll-only — it NEVER takes the daemon down.
- *
- * The 10-minute `--poll-remote-ms` pass remains the correctness backstop for
- * every path here; event delivery is best-effort by design.
- */
-
-import {
-  SQSClient,
-  ReceiveMessageCommand,
-  DeleteMessageCommand,
-} from "@aws-sdk/client-sqs";
-
-import { HttpPushTransport, type AuthTokenSource } from "./push-transport.js";
-import {
-  SqsPushReceiver,
-  type PushReceiver,
-  type PublishMetricFn,
-  type SqsClientLike,
-  type SyncEngineFn,
-} from "./push-receiver.js";
-import { PushEventEmitter } from "../watcher.js";
-import type { TreeChangeBatch } from "../watcher.js";
-import {
-  emitCloudTelemetry,
-  type CloudTelemetryClient,
-  type TelemetryClaims,
-} from "../telemetry-events.js";
-import {
-  createSyncLatencyMetricPublisher,
-  type SyncMetricCredentials,
-} from "./metrics.js";
-import { isEventSyncLocallyDisabled } from "./feature-flags.js";
-
-/**
- * Decide whether the client may request event-sync authority from the server.
- *
- * This is negative-only. An unset value or a former force-on value merely
- * permits the authenticated server request; it cannot authorize event sync.
- */
-export function resolveEventSync(override: string | undefined): boolean {
-  return !isEventSyncLocallyDisabled(override);
-}
-
-// ─── US-018: subscribe + credentials ────────────────────────────────────────
-
-export interface SubscribeSyncCredentials {
-  accessKeyId: string;
-  secretAccessKey: string;
-  sessionToken: string;
-  /** ISO8601 expiry of the vended STS credentials. */
-  expiration: string;
-}
-
-export interface SubscribeSyncResponse {
-  /** The caller's own per-device queue URL (stable across calls). */
-  queueUrl: string;
-  /** Region the queue lives in. */
-  region: string;
-  /** Short-lived creds scoped to receive/delete on exactly this queue. */
-  credentials: SubscribeSyncCredentials;
-  /** Optional short-lived creds scoped to publish sync latency CloudWatch metrics. */
-  cloudWatch?: {
-    region: string;
-    credentials: SyncMetricCredentials;
-  };
-}
-
-/** Minimal fetch seam (matches push-transport.ts's FetchLike posture). */
-type FetchLike = (
-  url: string,
-  init: {
-    method: string;
-    headers: Record<string, string>;
-    body: string;
-    signal?: AbortSignal;
-  },
-) => Promise<{ ok: boolean; status: number; text(): Promise<string> }>;
-
-function asNonEmptyString(v: unknown, field: string): string {
-  if (typeof v !== "string" || v.length === 0) {
-    throw new Error(`subscribeSyncReceive: response missing ${field}`);
-  }
-  return v;
-}
-
-function parseOptionalCloudWatch(
-  obj: Record<string, unknown>,
-): SubscribeSyncResponse["cloudWatch"] {
-  if (obj.cloudWatch === undefined) return undefined;
-  if (
-    !obj.cloudWatch ||
-    typeof obj.cloudWatch !== "object" ||
-    Array.isArray(obj.cloudWatch)
-  ) {
-    throw new Error("subscribeSyncReceive: response missing cloudWatch.region");
-  }
-  const cloudWatch = obj.cloudWatch as Record<string, unknown>;
-  const creds = (cloudWatch.credentials ?? {}) as Record<string, unknown>;
-  return {
-    region: asNonEmptyString(cloudWatch.region, "cloudWatch.region"),
-    credentials: {
-      accessKeyId: asNonEmptyString(
-        creds.accessKeyId,
-        "cloudWatch.credentials.accessKeyId",
-      ),
-      secretAccessKey: asNonEmptyString(
-        creds.secretAccessKey,
-        "cloudWatch.credentials.secretAccessKey",
-      ),
-      sessionToken:
-        typeof creds.sessionToken === "string" && creds.sessionToken.length > 0
-          ? creds.sessionToken
-          : undefined,
-      expiration:
-        typeof creds.expiration === "string" && creds.expiration.length > 0
-          ? creds.expiration
-          : undefined,
-    },
-  };
-}
-
-/**
- * `POST /v1/sync/subscribe` — provision (idempotently) this device's queue
- * and vend fresh receive credentials. Auth mirrors HttpPushTransport: Bearer
- * token resolved per-call via the supplied source.
- */
-export async function subscribeSyncReceive(opts: {
-  apiUrl: string;
-  authToken: AuthTokenSource;
-  deviceId: string;
-  timeoutMs?: number;
-  fetchImpl?: FetchLike;
-}): Promise<SubscribeSyncResponse> {
-  const apiUrl = opts.apiUrl.replace(/\/+$/, "");
-  const tok = opts.authToken;
-  const token = typeof tok === "function" ? await tok() : tok;
-  const fetchImpl =
-    opts.fetchImpl ??
-    ((input, init) => (globalThis.fetch as unknown as FetchLike)(input, init));
-
-  const controller = new AbortController();
-  const timeout = setTimeout(
-    () => controller.abort(),
-    opts.timeoutMs ?? 15_000,
-  );
-  let res: Awaited<ReturnType<FetchLike>>;
-  try {
-    res = await fetchImpl(`${apiUrl}/v1/sync/subscribe`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${token}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ deviceId: opts.deviceId }),
-      signal: controller.signal,
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => "");
-    throw new Error(
-      `subscribeSyncReceive: POST /v1/sync/subscribe failed (${res.status}): ${text.slice(0, 300)}`,
-    );
-  }
-  const parsed: unknown = JSON.parse(await res.text());
-  const obj = parsed as Record<string, unknown>;
-  const creds = (obj.credentials ?? {}) as Record<string, unknown>;
-  const cloudWatch = parseOptionalCloudWatch(obj);
-  const response: SubscribeSyncResponse = {
-    queueUrl: asNonEmptyString(obj.queueUrl, "queueUrl"),
-    region: asNonEmptyString(obj.region, "region"),
-    credentials: {
-      accessKeyId: asNonEmptyString(creds.accessKeyId, "credentials.accessKeyId"),
-      secretAccessKey: asNonEmptyString(
-        creds.secretAccessKey,
-        "credentials.secretAccessKey",
-      ),
-      sessionToken: asNonEmptyString(
-        creds.sessionToken,
-        "credentials.sessionToken",
-      ),
-      expiration: asNonEmptyString(creds.expiration, "credentials.expiration"),
-    },
-  };
-  if (cloudWatch !== undefined) response.cloudWatch = cloudWatch;
-  return response;
-}
-
-// ─── SQS SDK adapter ────────────────────────────────────────────────────────
-
-/**
- * Adapt the AWS SDK `SQSClient` to the receiver's narrow {@link SqsClientLike}
- * seam (the doc-promised `sqsClientFromAwsSdk` from push-receiver.ts). The
- * abort signal is forwarded so `dispose()` can cut a 20s long-poll short.
- */
-export function sqsClientFromAwsSdk(
-  client: Pick<SQSClient, "send">,
-): SqsClientLike {
-  return {
-    async receiveMessage({ queueUrl, maxMessages, waitTimeSeconds, signal }) {
-      const out = await client.send(
-        new ReceiveMessageCommand({
-          QueueUrl: queueUrl,
-          MaxNumberOfMessages: maxMessages,
-          WaitTimeSeconds: waitTimeSeconds,
-        }),
-        { abortSignal: signal },
-      );
-      return {
-        messages: (out.Messages ?? []).map((m) => ({
-          Body: m.Body,
-          ReceiptHandle: m.ReceiptHandle,
-          MessageId: m.MessageId,
-        })),
-      };
-    },
-    async deleteMessage({ queueUrl, receiptHandle }) {
-      await client.send(
-        new DeleteMessageCommand({
-          QueueUrl: queueUrl,
-          ReceiptHandle: receiptHandle,
-        }),
-      );
-    },
-  };
-}
-
-// ─── Refreshing SQS client (credential lifecycle) ───────────────────────────
-
-/**
- * Error names the SQS SDK surfaces when STS session creds have expired or
- * become invalid. Used for the reactive retry-once path; the proactive
- * skew-window refresh below should make these rare.
- */
-const EXPIRY_ERROR_NAMES = new Set([
-  "ExpiredToken",
-  "ExpiredTokenException",
-  "InvalidSecurityToken",
-  "InvalidClientTokenId",
-  "UnrecognizedClientException",
-]);
-
-function isExpiryError(err: unknown): boolean {
-  return err instanceof Error && EXPIRY_ERROR_NAMES.has(err.name);
-}
-
-/** Re-vend this long before the recorded expiry (proactive refresh window). */
-const REFRESH_SKEW_MS = 120_000;
-
-export interface RefreshingSqsClientOptions {
-  /** The initial subscribe response (creds + region + queue URL). */
-  initial: SubscribeSyncResponse;
-  /** Re-vend: called when creds are near/past expiry. Idempotent server-side. */
-  subscribe: () => Promise<SubscribeSyncResponse>;
-  /**
-   * Build the underlying narrow client from a subscribe response. Default:
-   * AWS SDK `SQSClient` via {@link sqsClientFromAwsSdk}. Tests inject a fake.
-   */
-  buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
-  /** Clock seam (tests). Default `Date.now`. */
-  now?: () => number;
-}
-
-function defaultBuildSqs(resp: SubscribeSyncResponse): SqsClientLike {
-  return sqsClientFromAwsSdk(
-    new SQSClient({
-      region: resp.region,
-      credentials: {
-        accessKeyId: resp.credentials.accessKeyId,
-        secretAccessKey: resp.credentials.secretAccessKey,
-        sessionToken: resp.credentials.sessionToken,
-        expiration: new Date(resp.credentials.expiration),
-      },
-    }),
-  );
-}
-
-/**
- * An {@link SqsClientLike} that owns the vended-credential lifecycle:
- *
- *  - PROACTIVE: before each call, if the recorded expiry is within the skew
- *    window, re-subscribe (re-vend) and rebuild the inner client first.
- *  - REACTIVE: if a call still fails with an expiry-class error (clock skew,
- *    revocation), re-vend once and retry the call once. Anything else — or a
- *    second failure — propagates to the receiver's own backoff/reconnect
- *    loop, whose retention-backed redelivery makes the miss recoverable.
- *
- * Concurrent refreshes collapse onto one in-flight subscribe promise.
- */
-export function createRefreshingSqsClient(
-  opts: RefreshingSqsClientOptions,
-): SqsClientLike {
-  const now = opts.now ?? (() => Date.now());
-  const buildSqs = opts.buildSqs ?? defaultBuildSqs;
-
-  let inner = buildSqs(opts.initial);
-  let expiresAtMs = Date.parse(opts.initial.credentials.expiration);
-  let refreshing: Promise<void> | null = null;
-
-  const refresh = (): Promise<void> => {
-    refreshing ??= (async () => {
-      try {
-        const resp = await opts.subscribe();
-        inner = buildSqs(resp);
-        expiresAtMs = Date.parse(resp.credentials.expiration);
-      } finally {
-        refreshing = null;
-      }
-    })();
-    return refreshing;
-  };
-
-  const ensureFresh = async (): Promise<void> => {
-    if (Number.isFinite(expiresAtMs) && now() >= expiresAtMs - REFRESH_SKEW_MS) {
-      await refresh();
-    }
-  };
-
-  const withRetry = async <T>(call: () => Promise<T>): Promise<T> => {
-    await ensureFresh();
-    try {
-      return await call();
-    } catch (err) {
-      if (!isExpiryError(err)) throw err;
-      await refresh();
-      return await call();
-    }
-  };
-
-  return {
-    receiveMessage: (args) => withRetry(() => inner.receiveMessage(args)),
-    deleteMessage: (args) => withRetry(() => inner.deleteMessage(args)),
-  };
-}
-
-// ─── US-017 + US-018: full wiring ───────────────────────────────────────────
-
-/** Structured line logger seam — the runner passes its stderr logger. */
-export type EventSyncLog = (message: string) => void;
-
-export interface StartEventSyncOptions {
-  hqRoot: string;
-  /** Vault API base URL (the runner's DEFAULT_VAULT_API_URL). */
-  apiUrl: string;
-  /** Cognito access-token source (getter for long-running daemons). */
-  authToken: AuthTokenSource;
-  /** This device's stable id (getOrCreateMachineId). */
-  deviceId: string;
-  /**
-   * Resolve the caller's tenant id (canonical person `prs_*` uid). The server
-   * rejects publishes whose `originTenantId` mismatches the JWT principal, so
-   * this MUST be the same identity the JWT resolves to.
-   */
-  resolveTenantId: () => Promise<string>;
-  /**
-   * The already-routed targeted-pull bridge (the runner's receiverSyncFn,
-   * funneled through its runGuarded mutex). Self-echo filtering happens HERE,
-   * before this is invoked.
-   */
-  syncFn: SyncEngineFn;
-  /** Diagnostic logger (one line per lifecycle event). Default: console.error. */
-  log?: EventSyncLog;
-  // ── test seams ──
-  subscribe?: (deviceId: string) => Promise<SubscribeSyncResponse>;
-  buildSqs?: (resp: SubscribeSyncResponse) => SqsClientLike;
-  transport?: HttpPushTransport;
-  telemetryClient?: CloudTelemetryClient | null;
-  telemetryClaims?: TelemetryClaims | null;
-  now?: () => number;
-}
-
-export interface EventSyncHandles {
-  /**
-   * Publish PushEvents for a settled change batch. Called by the runner
-   * AFTER the targeted push pass succeeds — an event must never announce
-   * bytes that are not in S3 yet. Fire-and-forget: failures are logged by
-   * the emitter's onError and the cadence poll covers the miss.
-   */
-  publishBatch: (batch: TreeChangeBatch) => void;
-  /** The live receiver (already started). */
-  receiver: PushReceiver;
-  /** This device's id — the runner uses it nowhere else, exposed for logs. */
-  ownDeviceId: string;
-  /** Tear down transport + receiver (runner shutdown path). */
-  dispose: () => Promise<void>;
-}
-
-/**
- * Bring up the publish + receive legs. Returns `null` (poll-only degradation)
- * on ANY startup failure — subscribe 5xx, tenant resolution failure, etc. —
- * after logging the reason. The caller treats `null` as "today's behavior".
- */
-export async function startEventSync(
-  opts: StartEventSyncOptions,
-): Promise<EventSyncHandles | null> {
-  const log = opts.log ?? ((m: string) => console.error(m));
-  try {
-    const tenantId = await opts.resolveTenantId();
-    const deviceId = opts.deviceId;
-
-    // The subscription response is the authenticated legacy inventory/grant.
-    // No email, environment, tenant, or queued hint can reach the live path
-    // without this server-issued queue and credential material.
-    const subscribe =
-      opts.subscribe ??
-      ((dev: string) =>
-        subscribeSyncReceive({
-          apiUrl: opts.apiUrl,
-          authToken: opts.authToken,
-          deviceId: dev,
-        }));
-    const initial = await subscribe(deviceId);
-
-    // ── publish leg (US-017) ──
-    const transport =
-      opts.transport ??
-      new HttpPushTransport({
-        apiUrl: opts.apiUrl,
-        pushPath: "/v1/sync/push",
-        authToken: opts.authToken,
-      });
-    await transport.start();
-    const emitter = new PushEventEmitter({
-      originTenantId: tenantId,
-      originDeviceId: deviceId,
-      transport,
-      // This adapter represents the authenticated server grant above. It is
-      // created only after the grant succeeds and ignores the tenant argument;
-      // it is not a tenant-derived client selector.
-      flagProvider: { isEnabled: () => true },
-      // Wall-clock sequence numbers: monotonic per device ACROSS daemon
-      // restarts (the emitter's default in-process counter restarts at 0,
-      // which would make peers' highest-seq dedupe drop every post-restart
-      // event). Receiver dedupe is per-path highest-seq, so ms-epoch values
-      // also stay comparable when two devices touch the same path.
-      getSequenceNumber: () => (opts.now ?? Date.now)(),
-      onError: (err, ctx) =>
-        log(
-          `event-sync: publish failed for ${ctx.relativePath ?? "?"}: ${err.message}`,
-        ),
-      telemetryClient: opts.telemetryClient,
-      telemetryClaims: opts.telemetryClaims,
-    });
-
-    // ── receive leg (US-018) ──
-    const publishMetric: PublishMetricFn = initial.cloudWatch
-      ? createSyncLatencyMetricPublisher(initial.cloudWatch)
-      : async () => undefined;
-    const sqs = createRefreshingSqsClient({
-      initial,
-      subscribe: () => subscribe(deviceId),
-      buildSqs: opts.buildSqs,
-      now: opts.now,
-    });
-    const receiver = new SqsPushReceiver({
-      tenantId,
-      queueUrl: initial.queueUrl,
-      sqs,
-      // Self-echo filter: this device's own publishes fan out to its own
-      // queue too — pulling them back is wasted work at best and a conflict
-      // source at worst. Skip before bridging into the sync engine.
-      syncFn: async (ctx) => {
-        if (ctx.event.originDeviceId === deviceId) return;
-        await opts.syncFn(ctx);
-      },
-      publishMetric,
-    });
-    await receiver.start();
-
-    log(
-      `event-sync: live (tenant=${tenantId} device=${deviceId} queue=${initial.queueUrl})`,
-    );
-    void emitCloudTelemetry(
-      opts.telemetryClient,
-      {
-        eventName: "event_sync_live",
-        source: "event-sync",
-        properties: {
-          deviceId,
-          startupResult: "live",
-          receiveState: "enabled",
-          publishState: "enabled",
-        },
-      },
-      { claims: opts.telemetryClaims },
-    );
-
-    return {
-      publishBatch: (batch) => void emitter.emitForBatch(batch),
-      receiver,
-      ownDeviceId: deviceId,
-      dispose: async () => {
-        await receiver.dispose().catch(() => undefined);
-        await transport.dispose().catch(() => undefined);
-      },
-    };
-  } catch (err) {
-    log(
-      `event-sync: startup failed, continuing poll-only: ${err instanceof Error ? err.message : String(err)}`,
-    );
-    void emitCloudTelemetry(
-      opts.telemetryClient,
-      {
-        eventName: "event_sync_live",
-        source: "event-sync",
-        properties: {
-          deviceId: opts.deviceId,
-          startupResult: "poll_only",
-          receiveState: "disabled",
-          publishState: "disabled",
-        },
-      },
-      { claims: opts.telemetryClaims },
-    );
-    return null;
-  }
-}