@indigoai-us/hq-cloud 5.26.0 → 5.28.0

package/src/sync/metrics.test.ts

@@ -0,0 +1,380 @@
+/**
+ * US-011 — unit tests for `src/sync/metrics.ts` + the receive-success-path
+ * metric emission wired into {@link SqsPushReceiver}.
+ *
+ * Mirrors the hq-pro PR #112 metrics test:
+ *   - mock the CloudWatchClient via `aws-sdk-client-mock`
+ *   - inject the mocked client via the `_setSyncCloudWatchClient` test seam
+ *   - assert namespace, metric name, unit, dimensions, value
+ *   - assert SDK errors are CAUGHT (the receive loop must never crash on a
+ *     CloudWatch outage)
+ *
+ * Plus the US-011-specific receiver wiring:
+ *   - on the receive-SUCCESS path, the receiver publishes exactly one latency
+ *     datum carrying the event's tenantId + sequenceNumber (mocked publish seam
+ *     — no real AWS)
+ *   - a failing publish seam does NOT crash the receiver loop
+ */
+
+import { Writable } from "node:stream";
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { mockClient } from "aws-sdk-client-mock";
+import {
+  CloudWatchClient,
+  PutMetricDataCommand,
+} from "@aws-sdk/client-cloudwatch";
+
+import {
+  SYNC_LATENCY_METRIC_NAME,
+  SYNC_METRIC_NAMESPACE,
+  _setSyncCloudWatchClient,
+  publishSyncLatencyMetric,
+  type SyncLatencyMetric,
+} from "./metrics.js";
+import { createLogger } from "./logger.js";
+import { encodePushEvent, type PushEvent } from "./push-event.js";
+import {
+  SqsPushReceiver,
+  type PublishMetricFn,
+  type SqsClientLike,
+  type SqsMessageLike,
+} from "./push-receiver.js";
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+function makeMetric(overrides: Partial<SyncLatencyMetric> = {}): SyncLatencyMetric {
+  return {
+    tenantId: "tenant-A",
+    relativePath: "docs/overview.md",
+    sequenceNumber: 42,
+    latencySeconds: 1.234,
+    timestamp: new Date("2026-05-19T12:00:00.000Z"),
+    ...overrides,
+  };
+}
+
+function captureStream(): { stream: Writable; lines: () => unknown[] } {
+  const chunks: string[] = [];
+  const stream = new Writable({
+    write(chunk, _enc, cb) {
+      chunks.push(
+        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8"),
+      );
+      cb();
+    },
+  });
+  return {
+    stream,
+    lines: () =>
+      chunks
+        .join("")
+        .split("\n")
+        .filter((s) => s.length > 0)
+        .map((s) => JSON.parse(s) as unknown),
+  };
+}
+
+const TENANT = "tenant-indigo";
+const QUEUE_URL =
+  "https://sqs.us-east-1.amazonaws.com/123456789012/sync-push-indigo-deviceB";
+
+function makeEvent(overrides: Partial<PushEvent> = {}): PushEvent {
+  return {
+    relativePath: "companies/indigo/notes.md",
+    contentHash:
+      "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    mtime: "2026-05-21T12:34:56.000Z",
+    originDeviceId: "device-A",
+    originTenantId: TENANT,
+    sequenceNumber: 7,
+    eventTimestamp: "2026-05-21T12:34:56.000Z",
+    ...overrides,
+  };
+}
+
+/** Minimal fake SQS that drains one batch then idles until abort. */
+class OneBatchSqs implements SqsClientLike {
+  private batch: SqsMessageLike[] | null;
+  readonly deleted: string[] = [];
+
+  constructor(messages: SqsMessageLike[]) {
+    this.batch = messages;
+  }
+
+  async receiveMessage(args: {
+    queueUrl: string;
+    maxMessages: number;
+    waitTimeSeconds: number;
+    signal: AbortSignal;
+  }): Promise<{ messages: SqsMessageLike[] }> {
+    if (this.batch) {
+      const b = this.batch;
+      this.batch = null;
+      return { messages: b };
+    }
+    return new Promise((resolve) => {
+      if (args.signal.aborted) return resolve({ messages: [] });
+      const t = setTimeout(() => resolve({ messages: [] }), 5);
+      (t as { unref?: () => void }).unref?.();
+      args.signal.addEventListener(
+        "abort",
+        () => {
+          clearTimeout(t);
+          resolve({ messages: [] });
+        },
+        { once: true },
+      );
+    });
+  }
+
+  async deleteMessage(args: {
+    queueUrl: string;
+    receiptHandle: string;
+  }): Promise<void> {
+    this.deleted.push(args.receiptHandle);
+  }
+}
+
+/** Wait until `predicate()` is true (polling), or throw after `timeoutMs`. */
+async function until(
+  predicate: () => boolean,
+  timeoutMs = 1000,
+): Promise<void> {
+  const start = Date.now();
+  while (!predicate()) {
+    if (Date.now() - start > timeoutMs) {
+      throw new Error("until() timed out");
+    }
+    await new Promise((r) => setTimeout(r, 5));
+  }
+}
+
+// ── publishSyncLatencyMetric (CloudWatch contract) ───────────────────────────
+
+const cwMock = mockClient(CloudWatchClient);
+
+describe("publishSyncLatencyMetric", () => {
+  beforeEach(() => {
+    cwMock.reset();
+    _setSyncCloudWatchClient(cwMock as unknown as CloudWatchClient);
+  });
+
+  it("sends a single PutMetricData with the documented namespace + metric name", async () => {
+    cwMock.on(PutMetricDataCommand).resolves({});
+
+    await publishSyncLatencyMetric(makeMetric());
+
+    const calls = cwMock.commandCalls(PutMetricDataCommand);
+    expect(calls).toHaveLength(1);
+
+    const input = calls[0]!.args[0].input;
+    expect(input.Namespace).toBe(SYNC_METRIC_NAMESPACE);
+    expect(input.Namespace).toBe("HQPro/Sync");
+    expect(input.MetricData).toHaveLength(1);
+    expect(input.MetricData![0].MetricName).toBe(SYNC_LATENCY_METRIC_NAME);
+    expect(input.MetricData![0].MetricName).toBe(
+      "hq-cloud.sync.p95_latency_seconds",
+    );
+  });
+
+  it("uses Unit=Seconds and stamps the supplied timestamp", async () => {
+    cwMock.on(PutMetricDataCommand).resolves({});
+
+    const ts = new Date("2026-05-19T07:00:00.000Z");
+    await publishSyncLatencyMetric(makeMetric({ timestamp: ts }));
+
+    const datum = cwMock.commandCalls(PutMetricDataCommand)[0]!.args[0].input
+      .MetricData![0];
+    expect(datum.Unit).toBe("Seconds");
+    expect(datum.Timestamp).toEqual(ts);
+  });
+
+  it("publishes the observed latencySeconds value verbatim", async () => {
+    cwMock.on(PutMetricDataCommand).resolves({});
+
+    await publishSyncLatencyMetric(makeMetric({ latencySeconds: 2.71828 }));
+
+    const datum = cwMock.commandCalls(PutMetricDataCommand)[0]!.args[0].input
+      .MetricData![0];
+    expect(datum.Value).toBe(2.71828);
+  });
+
+  it("attaches a `TenantId` dimension carrying the tenantId", async () => {
+    cwMock.on(PutMetricDataCommand).resolves({});
+
+    await publishSyncLatencyMetric(makeMetric({ tenantId: "tenant-prs_xyz" }));
+
+    const datum = cwMock.commandCalls(PutMetricDataCommand)[0]!.args[0].input
+      .MetricData![0];
+    expect(datum.Dimensions).toEqual([
+      { Name: "TenantId", Value: "tenant-prs_xyz" },
+    ]);
+  });
+
+  it("does not throw when the CloudWatch SDK rejects (sync loop must not crash)", async () => {
+    cwMock
+      .on(PutMetricDataCommand)
+      .rejects(new Error("CloudWatch unavailable"));
+    const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    await expect(
+      publishSyncLatencyMetric(makeMetric()),
+    ).resolves.toBeUndefined();
+
+    expect(consoleSpy).toHaveBeenCalledWith(
+      "Failed to publish sync latency metric to CloudWatch:",
+      "CloudWatch unavailable",
+    );
+
+    consoleSpy.mockRestore();
+  });
+
+  it("routes SDK failures through the supplied pino logger when provided", async () => {
+    cwMock.on(PutMetricDataCommand).rejects(new Error("ThrottlingException"));
+    const { stream, lines } = captureStream();
+    const logger = createLogger({
+      component: "metrics-test",
+      destination: stream,
+      level: "debug",
+    });
+
+    await publishSyncLatencyMetric(
+      makeMetric({ sequenceNumber: 99, relativePath: "a.md" }),
+      { logger },
+    );
+
+    const failureLine = (lines() as Array<{ event?: string }>).find(
+      (l) => l.event === "sync.metric.publish_failed",
+    );
+    expect(failureLine).toBeDefined();
+    expect(failureLine).toMatchObject({
+      event: "sync.metric.publish_failed",
+      tenantId: "tenant-A",
+      relativePath: "a.md",
+      sequenceNumber: 99,
+    });
+  });
+
+  it("accepts a per-call client override (does not call the singleton)", async () => {
+    cwMock
+      .on(PutMetricDataCommand)
+      .rejects(new Error("singleton should not be used"));
+
+    const overrideMock = mockClient(CloudWatchClient);
+    overrideMock.on(PutMetricDataCommand).resolves({});
+
+    await publishSyncLatencyMetric(makeMetric(), {
+      client: overrideMock as unknown as CloudWatchClient,
+    });
+
+    expect(overrideMock.commandCalls(PutMetricDataCommand)).toHaveLength(1);
+    expect(cwMock.commandCalls(PutMetricDataCommand)).toHaveLength(0);
+
+    overrideMock.restore();
+  });
+});
+
+// ── Receive-success-path metric emission (US-011 AC: "metric emission unit-
+//    tested on the receive success path") ──────────────────────────────────
+
+describe("SqsPushReceiver metric emission on the receive-success path", () => {
+  let receiver: SqsPushReceiver | undefined;
+
+  afterEach(async () => {
+    if (receiver) {
+      await receiver.dispose();
+      receiver = undefined;
+    }
+  });
+
+  it("publishes exactly one latency datum per processed event (mocked publish, no real AWS)", async () => {
+    const published: SyncLatencyMetric[] = [];
+    const publishMetric: PublishMetricFn = async (m) => {
+      published.push(m);
+    };
+    const event = makeEvent({ sequenceNumber: 7 });
+
+    receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: QUEUE_URL,
+      sqs: new OneBatchSqs([{ Body: encodePushEvent(event), ReceiptHandle: "rh-7" }]),
+      syncFn: async () => {
+        /* successful pull */
+      },
+      enabled: true,
+      publishMetric,
+      // Fake clock: start measured at T, success at T+1500ms → 1.5s latency.
+      now: (() => {
+        let t = Date.parse(event.eventTimestamp);
+        return () => {
+          const cur = t;
+          t += 1500;
+          return cur;
+        };
+      })(),
+    });
+
+    await receiver.start();
+    await until(() => published.length >= 1);
+
+    expect(published).toHaveLength(1);
+    expect(published[0]).toMatchObject({
+      tenantId: TENANT,
+      relativePath: event.relativePath,
+      sequenceNumber: 7,
+    });
+    expect(published[0]!.latencySeconds).toBeGreaterThanOrEqual(0);
+    expect(Number.isFinite(published[0]!.latencySeconds)).toBe(true);
+  });
+
+  it("does NOT publish a metric when the syncFn throws (failures don't skew p95)", async () => {
+    const published: SyncLatencyMetric[] = [];
+    const event = makeEvent({ sequenceNumber: 8 });
+
+    receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: QUEUE_URL,
+      sqs: new OneBatchSqs([{ Body: encodePushEvent(event), ReceiptHandle: "rh-8" }]),
+      syncFn: async () => {
+        throw new Error("pull failed");
+      },
+      enabled: true,
+      publishMetric: async (m) => {
+        published.push(m);
+      },
+    });
+
+    await receiver.start();
+    // Give the loop a few ticks to process the message + (not) publish.
+    await new Promise((r) => setTimeout(r, 50));
+
+    expect(published).toHaveLength(0);
+    expect(receiver.processedCount).toBe(0);
+  });
+
+  it("a throwing publish seam does not crash the receiver loop", async () => {
+    const event = makeEvent({ sequenceNumber: 9 });
+    let synced = false;
+
+    receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: QUEUE_URL,
+      sqs: new OneBatchSqs([{ Body: encodePushEvent(event), ReceiptHandle: "rh-9" }]),
+      syncFn: async () => {
+        synced = true;
+      },
+      enabled: true,
+      publishMetric: async () => {
+        throw new Error("metric backend down");
+      },
+    });
+
+    await receiver.start();
+    await until(() => synced);
+
+    // The sync still completed and the receiver is still alive (connected).
+    expect(receiver.processedCount).toBe(1);
+    expect(receiver.connected).toBe(true);
+  });
+});

package/src/sync/metrics.ts

@@ -0,0 +1,158 @@
+/**
+ * Sync pipeline metrics — CloudWatch custom metrics for the event-driven push
+ * receive loop (project event-driven-sync-menubar US-011).
+ *
+ * Publishes one `hq-cloud.sync.p95_latency_seconds` datum per successfully
+ * processed push event to the `HQPro/Sync` namespace, with a `TenantId`
+ * dimension so the dashboard widget can be filtered per tenant. CloudWatch
+ * aggregates p50/p95/p99 across the time window via the dashboard's
+ * `Statistics` setting; the receive-loop side just emits one raw value per
+ * event.
+ *
+ * Best-effort emission
+ * ────────────────────
+ *   - module-level singleton CloudWatchClient with a `_setSyncCloudWatchClient`
+ *     test seam
+ *   - `publishSyncLatencyMetric` catches all errors and NEVER throws — a
+ *     CloudWatch outage MUST NOT crash the sync receive loop. The cadence
+ *     safety net still picks up missed work and metric blanks are recoverable;
+ *     a crashed loop is not.
+ *   - explicit `Unit` + `Timestamp` + `Dimensions` per datum
+ *
+ * Why latency (and not also "events received" / "events failed")?
+ * ───────────────────────────────────────────────────────────────
+ * US-011 AC#1 calls for the p95 latency metric specifically. The receive-loop's
+ * existing `processedCount` / log lines cover the count/failure dimensions for
+ * now; widening to additional metric names happens in a follow-up if the
+ * operator dashboard grows.
+ *
+ * Adapted from indigoai-us/hq-pro PR #112 (src/sync/metrics.ts) into
+ * @indigoai-us/hq-cloud (Path B).
+ */
+
+import {
+  CloudWatchClient,
+  PutMetricDataCommand,
+  type MetricDatum,
+} from "@aws-sdk/client-cloudwatch";
+import type { Logger } from "pino";
+
+// ── Constants ──────────────────────────────────────────────────────────────
+
+/**
+ * CloudWatch metric namespace for the sync pipeline. Matches the server-side
+ * namespace (hq-pro PR #112) so the dashboard + alarm cover the client path.
+ */
+export const SYNC_METRIC_NAMESPACE = "HQPro/Sync";
+
+/**
+ * Metric name for per-event sync latency in seconds. The dashboard widget
+ * applies `Statistics: ["p95"]` to aggregate across the time window — the
+ * receive loop just emits one raw `Seconds` value per processed event.
+ *
+ * Name chosen to match the PRD's alarm threshold (`p95 > 10s`).
+ */
+export const SYNC_LATENCY_METRIC_NAME = "hq-cloud.sync.p95_latency_seconds";
+
+// ── Types ──────────────────────────────────────────────────────────────────
+
+/**
+ * One latency observation. `latencySeconds` is the wall-clock duration from
+ * save-on-A to visible-on-B (or, on the receive loop, the `syncFn(ctx)`
+ * duration); we ONLY publish on the success path so failed syncs don't skew
+ * p95 toward infinity.
+ *
+ * `relativePath` and `sequenceNumber` are NOT used as CloudWatch dimensions
+ * (cardinality explosion) — they're captured here for the optional debug log
+ * emitted on failure so operators can correlate back to the 3-log chain when
+ * investigating a spike.
+ */
+export interface SyncLatencyMetric {
+  tenantId: string;
+  relativePath: string;
+  sequenceNumber: number;
+  latencySeconds: number;
+  timestamp: Date;
+}
+
+// ── Client ─────────────────────────────────────────────────────────────────
+
+let _cwClient: CloudWatchClient | undefined;
+
+function getCloudWatchClient(): CloudWatchClient {
+  if (!_cwClient) {
+    _cwClient = new CloudWatchClient({});
+  }
+  return _cwClient;
+}
+
+/**
+ * Replace the CloudWatch client (for testing).
+ * @internal
+ */
+export function _setSyncCloudWatchClient(client: CloudWatchClient): void {
+  _cwClient = client;
+}
+
+// ── Publish ────────────────────────────────────────────────────────────────
+
+export interface PublishSyncLatencyMetricOptions {
+  /** Override the CloudWatch client (tests). Defaults to the module singleton. */
+  client?: CloudWatchClient;
+  /** Optional pino logger for emission failures. */
+  logger?: Logger;
+}
+
+/**
+ * Publish a single latency datum to CloudWatch.
+ *
+ * Best-effort: any error from the SDK is CAUGHT and logged. A CloudWatch outage
+ * MUST NOT crash the sync receive loop — the cadence safety net still picks up
+ * missed work, and metric blanks are recoverable; a crashed loop is not.
+ *
+ * Dimension: `TenantId` only. The `relativePath`/`sequenceNumber` fields on the
+ * input are intentionally NOT promoted to dimensions (cardinality), but they
+ * ride along on the failure log so an operator can correlate a missed datum to
+ * its 3-log chain entry.
+ */
+export async function publishSyncLatencyMetric(
+  metric: SyncLatencyMetric,
+  opts: PublishSyncLatencyMetricOptions = {},
+): Promise<void> {
+  const datum: MetricDatum = {
+    MetricName: SYNC_LATENCY_METRIC_NAME,
+    Value: metric.latencySeconds,
+    Unit: "Seconds",
+    Timestamp: metric.timestamp,
+    Dimensions: [{ Name: "TenantId", Value: metric.tenantId }],
+  };
+
+  try {
+    const client = opts.client ?? getCloudWatchClient();
+    await client.send(
+      new PutMetricDataCommand({
+        Namespace: SYNC_METRIC_NAMESPACE,
+        MetricData: [datum],
+      }),
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (opts.logger) {
+      opts.logger.warn(
+        {
+          event: "sync.metric.publish_failed",
+          tenantId: metric.tenantId,
+          relativePath: metric.relativePath,
+          sequenceNumber: metric.sequenceNumber,
+          err: { message },
+        },
+        "failed to publish sync latency metric to CloudWatch",
+      );
+    } else {
+      console.error(
+        "Failed to publish sync latency metric to CloudWatch:",
+        message,
+      );
+    }
+  }
+}