@checkstack/healthcheck-backend 1.17.0 → 1.18.0

package/src/realtime-aggregation.ts

@@ -1,4 +1,7 @@
-import type { SafeDatabase, CollectorRegistry } from "@checkstack/backend-api";
+import type {
+  ScopedQueryRunner,
+  CollectorRegistry,
+} from "@checkstack/backend-api";
 import {
   ASSERTIONS_AGG_KEY,
   AssertionOutcomeSchema,
@@ -11,7 +14,11 @@ import * as schema from "./schema";
 import { healthCheckAggregates } from "./schema";
 import { eq, and, sql } from "drizzle-orm";
 
-type Db = SafeDatabase<typeof schema>;
+// Accepts either the scoped database OR a transaction handle from it, so the
+// caller can compose the aggregate SELECT + UPSERT inside a single batching
+// transaction (one `SET LOCAL search_path` for the whole write group) — see
+// `withScopedTransaction`.
+type Db = ScopedQueryRunner<typeof schema>;
 
 /**
  * Get the hour bucket start time for a given timestamp.

package/src/rollup-consumer.test.ts

@@ -0,0 +1,191 @@
+import { describe, it, expect, mock } from "bun:test";
+import {
+  setupRollupConsumer,
+  encodeRollupDebounceJobId,
+  HEALTH_ROLLUP_QUEUE,
+  type HealthRollupJobPayload,
+} from "./rollup-consumer";
+import { encodeHealthEntityId } from "./health-entity-id";
+import type { EntityChanged, OnEntityChanged } from "@checkstack/automation-backend";
+
+describe("encodeRollupDebounceJobId", () => {
+  it("coalesces changes in the same window to one jobId", () => {
+    const a = encodeRollupDebounceJobId({ systemId: "s1", now: 10_000, windowMs: 2000 });
+    const b = encodeRollupDebounceJobId({ systemId: "s1", now: 11_500, windowMs: 2000 });
+    expect(a).toBe(b);
+    expect(a).toBe("healthrollup:s1:5");
+  });
+
+  it("uses a fresh jobId for the next window", () => {
+    const a = encodeRollupDebounceJobId({ systemId: "s1", now: 10_000, windowMs: 2000 });
+    const b = encodeRollupDebounceJobId({ systemId: "s1", now: 12_500, windowMs: 2000 });
+    expect(a).not.toBe(b);
+  });
+
+  it("keys distinct systems separately", () => {
+    const a = encodeRollupDebounceJobId({ systemId: "s1", now: 10_000, windowMs: 2000 });
+    const b = encodeRollupDebounceJobId({ systemId: "s2", now: 10_000, windowMs: 2000 });
+    expect(a).not.toBe(b);
+  });
+});
+
+interface Harness {
+  enqueue: ReturnType<typeof mock>;
+  consumeHandler: (job: { data: HealthRollupJobPayload }) => Promise<void>;
+  changeHandler: (change: EntityChanged) => Promise<void>;
+  getSystemHealthStatus: ReturnType<typeof mock>;
+  broadcast: ReturnType<typeof mock>;
+  invalidateSystem: ReturnType<typeof mock>;
+}
+
+async function setup(opts: {
+  statuses?: string[]; // successive getSystemHealthStatus results
+  now?: number;
+} = {}): Promise<Harness> {
+  const statuses = opts.statuses ?? ["healthy", "healthy"];
+  let statusCall = 0;
+  const getSystemHealthStatus = mock(async () => ({
+    status: statuses[Math.min(statusCall++, statuses.length - 1)],
+    checkStatuses: [],
+  }));
+
+  const enqueue = mock(async () => "job-id");
+  let consumeHandler!: (job: { data: HealthRollupJobPayload }) => Promise<void>;
+  const queue = {
+    enqueue,
+    consume: mock(
+      async (
+        handler: (job: { data: HealthRollupJobPayload }) => Promise<void>,
+      ) => {
+        consumeHandler = handler;
+      },
+    ),
+  };
+  const queueManager = {
+    getQueue: mock((name: string) => {
+      expect(name).toBe(HEALTH_ROLLUP_QUEUE);
+      return queue;
+    }),
+  } as unknown as Parameters<typeof setupRollupConsumer>[0]["queueManager"];
+
+  let changeHandler!: (change: EntityChanged) => Promise<void>;
+  const onEntityChanged = mock((input: Parameters<OnEntityChanged>[0]) => {
+    changeHandler = input.handler as (c: EntityChanged) => Promise<void>;
+    return async () => {};
+  }) as unknown as OnEntityChanged;
+
+  const broadcast = mock(async () => {});
+  const invalidateSystem = mock(async () => {});
+
+  await setupRollupConsumer({
+    queueManager,
+    onEntityChanged,
+    service: { getSystemHealthStatus } as never,
+    advisoryLock: {
+      withXactLock: async ({ fn }: { fn: () => Promise<unknown> }) => fn(),
+    } as never,
+    signalService: { broadcast } as never,
+    cache: { invalidateSystem } as never,
+    // No entity handle: writeHealthEntity runs `apply` directly.
+    getHealthEntity: () => undefined,
+    logger: {
+      debug: () => {},
+      info: () => {},
+      warn: () => {},
+      error: () => {},
+    } as never,
+    now: () => opts.now ?? 10_000,
+  });
+
+  return {
+    enqueue,
+    consumeHandler,
+    changeHandler,
+    getSystemHealthStatus,
+    broadcast,
+    invalidateSystem,
+  };
+}
+
+describe("setupRollupConsumer subscription", () => {
+  it("enqueues a debounced rollup job for a per-env health change", async () => {
+    const h = await setup({ now: 10_000 });
+    await h.changeHandler({
+      kind: "health",
+      id: encodeHealthEntityId({ systemId: "s1", environmentId: "prod" }),
+      prev: null,
+      next: { status: "unhealthy" },
+    } as unknown as EntityChanged);
+
+    expect(h.enqueue).toHaveBeenCalledTimes(1);
+    expect(h.enqueue.mock.calls[0]![0]).toEqual({ systemId: "s1" });
+    expect(h.enqueue.mock.calls[0]![1]).toMatchObject({
+      jobId: "healthrollup:s1:5",
+      startDelay: 2,
+    });
+  });
+
+  it("IGNORES a bare-rollup change (feedback-loop guard)", async () => {
+    const h = await setup();
+    await h.changeHandler({
+      kind: "health",
+      id: encodeHealthEntityId({ systemId: "s1" }), // bare rollup id
+      prev: null,
+      next: { status: "unhealthy" },
+    } as unknown as EntityChanged);
+
+    expect(h.enqueue).not.toHaveBeenCalled();
+  });
+
+  it("coalesces two per-env changes in the same window to one jobId", async () => {
+    const h = await setup({ now: 10_000 });
+    const mk = (env: string) =>
+      ({
+        kind: "health",
+        id: encodeHealthEntityId({ systemId: "s1", environmentId: env }),
+        prev: null,
+        next: { status: "unhealthy" },
+      }) as unknown as EntityChanged;
+
+    await h.changeHandler(mk("prod"));
+    await h.changeHandler(mk("staging"));
+
+    // Both enqueue with the SAME jobId; the queue backend dedupes them.
+    expect(h.enqueue).toHaveBeenCalledTimes(2);
+    expect(h.enqueue.mock.calls[0]![1]).toMatchObject({
+      jobId: "healthrollup:s1:5",
+    });
+    expect(h.enqueue.mock.calls[1]![1]).toMatchObject({
+      jobId: "healthrollup:s1:5",
+    });
+  });
+});
+
+describe("setupRollupConsumer rollup recompute", () => {
+  it("broadcasts SYSTEM_STATUS_CHANGED + invalidates cache on a rollup status change", async () => {
+    const h = await setup({ statuses: ["healthy", "unhealthy"] });
+    await h.consumeHandler({ data: { systemId: "s1" } });
+
+    expect(h.getSystemHealthStatus).toHaveBeenCalled();
+    expect(h.invalidateSystem).toHaveBeenCalledWith("s1");
+    expect(h.broadcast).toHaveBeenCalledTimes(1);
+    const payload = h.broadcast.mock.calls[0]![1] as {
+      systemId: string;
+      previousStatus: string;
+      newStatus: string;
+    };
+    expect(payload).toEqual({
+      systemId: "s1",
+      previousStatus: "healthy",
+      newStatus: "unhealthy",
+    });
+  });
+
+  it("does NOT broadcast when the rollup status is unchanged", async () => {
+    const h = await setup({ statuses: ["degraded", "degraded"] });
+    await h.consumeHandler({ data: { systemId: "s1" } });
+
+    expect(h.broadcast).not.toHaveBeenCalled();
+    expect(h.invalidateSystem).not.toHaveBeenCalled();
+  });
+});

package/src/rollup-consumer.ts

@@ -0,0 +1,160 @@
+/**
+ * Event-driven system-rollup consumer (Phase 2 of the per-environment-jobs
+ * migration).
+ *
+ * Under per-environment jobs, each recurring job writes ONLY its own
+ * `"<systemId>::<environmentId>"` health entity; the bare `"<systemId>"` ROLLUP
+ * entity (the worst-status view every existing system-level consumer, badge,
+ * SLO rule and dashboard references) is no longer recomputed inline. This
+ * consumer closes that gap WITHOUT re-coupling the executor to the rollup:
+ *
+ *  1. It subscribes to `health` `ENTITY_CHANGED` with **work-queue** delivery
+ *     (exactly once per cluster per change) and filters to PER-ENV ids
+ *     (`environmentId !== null`). A bare-rollup change is IGNORED, so the
+ *     rollup write this consumer itself performs can never re-trigger it - no
+ *     feedback loop.
+ *  2. Per-env changes are DEBOUNCED per system into a short fixed window: the
+ *     handler enqueues a `{ systemId }` job on the rollup queue keyed
+ *     `healthrollup:<systemId>:<window-bucket>`. Both queue backends dedupe on
+ *     jobId, so a burst of env transitions for one system in the same window
+ *     coalesces to ONE rollup recompute (and the bucket id is used once, so
+ *     BullMQ's completed-job retention never drops a later window's job).
+ *  3. The rollup-queue consumer recomputes the bare `"<systemId>"` entity via
+ *     `recomputeSystemRollupHealth`, which diffs prev → next inside the
+ *     `health:<systemId>` advisory lock, emits the rollup `ENTITY_CHANGED`, and
+ *     (on a real change) invalidates the cache + broadcasts
+ *     `SYSTEM_STATUS_CHANGED`.
+ *
+ * Notifications are intentionally NOT sent here: each env run already notifies
+ * its own transition, so a rollup notification would duplicate it. This is the
+ * structural form of the #417 rollup-notification dedup - for a fanned-out
+ * system the rollup notification is ALWAYS suppressed; an env-less system needs
+ * no consumer at all (its run IS the bare-entity write and notifies directly).
+ *
+ * Scale-correctness (state-and-scale): the rollup is recomputed from Postgres
+ * (`getSystemHealthStatus`) on whichever pod claims the debounced job, and the
+ * advisory lock serializes concurrent recomputes, so every pod converges to the
+ * same rollup regardless of which one ran it.
+ */
+import type { Logger, AdvisoryLockService } from "@checkstack/backend-api";
+import type { QueueManager } from "@checkstack/queue-api";
+import type { SignalService } from "@checkstack/signal-common";
+import type {
+  OnEntityChanged,
+  EntityChanged,
+  EntityHandle,
+} from "@checkstack/automation-backend";
+import { HEALTH_ENTITY_KIND, type HealthEntityState } from "./health-entity";
+import { parseHealthEntityId } from "./health-entity-id";
+import { recomputeSystemRollupHealth } from "./queue-executor";
+import type { HealthCheckService } from "./service";
+import type { HealthCheckCache } from "./cache";
+
+/** The dedicated queue the debounced rollup recomputes run on. */
+export const HEALTH_ROLLUP_QUEUE = "health-rollup";
+
+/** Payload for a debounced rollup recompute job. */
+export interface HealthRollupJobPayload {
+  systemId: string;
+}
+
+/** Default debounce window (ms) a burst of per-env changes coalesces into. */
+export const DEFAULT_ROLLUP_DEBOUNCE_MS = 2000;
+
+/**
+ * The dedup jobId for a debounced rollup recompute. Keyed on the system AND a
+ * fixed time bucket so a burst within one window coalesces to a single job,
+ * while the NEXT window gets a fresh id (so a completed-job retention on the
+ * queue backend can never suppress a later window's recompute).
+ */
+export function encodeRollupDebounceJobId(props: {
+  systemId: string;
+  now: number;
+  windowMs: number;
+}): string {
+  const { systemId, now, windowMs } = props;
+  const bucket = Math.floor(now / windowMs);
+  return `healthrollup:${systemId}:${bucket}`;
+}
+
+export interface RollupConsumerDeps {
+  queueManager: QueueManager;
+  onEntityChanged: OnEntityChanged;
+  service: HealthCheckService;
+  advisoryLock: AdvisoryLockService;
+  signalService: SignalService;
+  cache: HealthCheckCache;
+  getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
+  logger: Logger;
+  /** Debounce window in ms. Defaults to {@link DEFAULT_ROLLUP_DEBOUNCE_MS}. */
+  debounceMs?: number;
+  /** Injectable clock for tests. Defaults to `Date.now`. */
+  now?: () => number;
+}
+
+/**
+ * Wire the rollup-queue consumer + the per-env `health` change subscription.
+ * Returns the `onEntityChanged` unsubscribe handle for teardown.
+ */
+export async function setupRollupConsumer(
+  deps: RollupConsumerDeps,
+): Promise<() => Promise<void>> {
+  const {
+    queueManager,
+    onEntityChanged,
+    service,
+    advisoryLock,
+    signalService,
+    cache,
+    getHealthEntity,
+    logger,
+    debounceMs = DEFAULT_ROLLUP_DEBOUNCE_MS,
+    now = () => Date.now(),
+  } = deps;
+
+  const rollupQueue =
+    queueManager.getQueue<HealthRollupJobPayload>(HEALTH_ROLLUP_QUEUE);
+
+  // Consumer: recompute the bare-system rollup entity for the job's system.
+  await rollupQueue.consume(
+    async (job) => {
+      await recomputeSystemRollupHealth({
+        systemId: job.data.systemId,
+        service,
+        getHealthEntity,
+        advisoryLock,
+        signalService,
+        cache,
+        logger,
+      });
+    },
+    { consumerGroup: "health-rollup" },
+  );
+
+  // Subscription: a per-env `health` change debounces a rollup recompute for
+  // its system. Bare-rollup changes are ignored (feedback-loop guard).
+  const unsubscribe = onEntityChanged({
+    kind: HEALTH_ENTITY_KIND,
+    delivery: { mode: "work-queue", workerGroup: "health-rollup-debounce" },
+    handler: async (change: EntityChanged) => {
+      const { systemId, environmentId } = parseHealthEntityId(change.id);
+      // Only PER-ENV changes drive a rollup recompute. A bare-`<systemId>`
+      // change is either an env-less run (already the rollup) or this
+      // consumer's own rollup write - never re-enqueue on it.
+      if (environmentId === null) return;
+
+      const jobId = encodeRollupDebounceJobId({
+        systemId,
+        now: now(),
+        windowMs: debounceMs,
+      });
+      await rollupQueue.enqueue(
+        { systemId },
+        { jobId, startDelay: Math.ceil(debounceMs / 1000) },
+      );
+    },
+  });
+
+  logger.debug("✅ Health rollup consumer wired (per-env → debounced rollup).");
+  return unsubscribe;
+}

package/src/router.ts

@@ -213,21 +213,18 @@ export const createHealthCheckRouter = (opts: {
   }) => {
     await cache.invalidateSystem(args.systemId);
 
-    // If enabling the health check, schedule it immediately so it starts
-    // probing right away.
+    // If enabling the health check, reconcile this system's per-env recurring
+    // jobs immediately so it starts probing right away. A system-scoped
+    // reconcile only adds/updates (no orphan cleanup), so it needs no lock.
     if (args.enabled) {
-      const config = await service.getConfiguration(args.configurationId);
-      if (config) {
-        const { scheduleHealthCheck } = await import("./queue-executor");
-        await scheduleHealthCheck({
-          queueManager: args.queueManager,
-          payload: {
-            configId: config.id,
-            systemId: args.systemId,
-          },
-          intervalSeconds: config.intervalSeconds,
-        });
-      }
+      const { reconcileHealthCheckJobs } = await import("./schedule-reconciler");
+      await reconcileHealthCheckJobs({
+        db: database,
+        queueManager: args.queueManager,
+        catalogClient,
+        logger,
+        systemId: args.systemId,
+      });
     }
 
     // Notify subscribers (e.g., satellite-backend) that assignments changed.

package/src/schedule-jitter.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from "bun:test";
+import {
+  computeScheduleJitterSeconds,
+  DEFAULT_JITTER_WINDOW_SECONDS,
+} from "./schedule-jitter";
+
+describe("computeScheduleJitterSeconds", () => {
+  it("is deterministic for the same key + interval", () => {
+    const a = computeScheduleJitterSeconds({ key: "sys:cfg", intervalSeconds: 60 });
+    const b = computeScheduleJitterSeconds({ key: "sys:cfg", intervalSeconds: 60 });
+    expect(a).toBe(b);
+  });
+
+  it("stays within [0, min(interval, window))", () => {
+    // Short interval: window is the interval itself.
+    for (let i = 0; i < 200; i++) {
+      const v = computeScheduleJitterSeconds({
+        key: `k${i}`,
+        intervalSeconds: 10,
+      });
+      expect(v).toBeGreaterThanOrEqual(0);
+      expect(v).toBeLessThan(10);
+    }
+    // Long interval: capped by the default window.
+    for (let i = 0; i < 200; i++) {
+      const v = computeScheduleJitterSeconds({
+        key: `k${i}`,
+        intervalSeconds: 3600,
+      });
+      expect(v).toBeGreaterThanOrEqual(0);
+      expect(v).toBeLessThan(DEFAULT_JITTER_WINDOW_SECONDS);
+    }
+  });
+
+  it("spreads a synchronized set across the window (de-clusters)", () => {
+    // 50 checks that share an interval must NOT all land on the same offset.
+    const offsets = new Set<number>();
+    for (let i = 0; i < 50; i++) {
+      offsets.add(
+        computeScheduleJitterSeconds({
+          key: `system-${i}:check-a`,
+          intervalSeconds: 60,
+        }),
+      );
+    }
+    // Comfortably more than a handful of distinct slots out of a 30s window.
+    expect(offsets.size).toBeGreaterThan(10);
+  });
+
+  it("returns 0 for a non-positive interval or window", () => {
+    expect(
+      computeScheduleJitterSeconds({ key: "k", intervalSeconds: 0 }),
+    ).toBe(0);
+    expect(
+      computeScheduleJitterSeconds({
+        key: "k",
+        intervalSeconds: 60,
+        maxWindowSeconds: 0,
+      }),
+    ).toBe(0);
+  });
+
+  it("different keys generally produce different offsets", () => {
+    const a = computeScheduleJitterSeconds({ key: "alpha", intervalSeconds: 60 });
+    const b = computeScheduleJitterSeconds({ key: "beta", intervalSeconds: 60 });
+    // Not a hard guarantee for any two strings, but these two must differ.
+    expect(a).not.toBe(b);
+  });
+});

package/src/schedule-jitter.ts

@@ -0,0 +1,50 @@
+/**
+ * Deterministic per-check scheduling jitter to de-cluster the health-check
+ * "thundering herd".
+ *
+ * Many checks created together - or all overdue at once on a fresh boot - would
+ * otherwise be scheduled with the same `startDelay` and identical intervals, so
+ * they fire on the same phase forever: dozens of TCP/TLS handshakes contending
+ * at the same instant, which inflates and destabilizes per-run connection setup
+ * (the observed "same check, same site, wildly different durations"). Offsetting
+ * each check's first fire by a stable fraction of its interval spreads them out;
+ * because the queue anchors the recurrence to that first fire, the offset
+ * persists for the schedule's whole life.
+ *
+ * The offset is DETERMINISTIC in the check's key (config + system), so a check
+ * keeps the same slot across restarts instead of re-clustering on a fresh random
+ * draw each boot.
+ */
+
+/** Default upper bound on the jitter window, in seconds. */
+export const DEFAULT_JITTER_WINDOW_SECONDS = 30;
+
+/**
+ * A stable jitter offset in `[0, min(intervalSeconds, maxWindowSeconds))`
+ * seconds, derived from `key`. Short intervals spread across the whole interval;
+ * long intervals cap the window so a check still starts reasonably promptly.
+ */
+export function computeScheduleJitterSeconds({
+  key,
+  intervalSeconds,
+  maxWindowSeconds = DEFAULT_JITTER_WINDOW_SECONDS,
+}: {
+  key: string;
+  intervalSeconds: number;
+  maxWindowSeconds?: number;
+}): number {
+  const window = Math.min(
+    Math.max(0, Math.floor(intervalSeconds)),
+    Math.max(0, Math.floor(maxWindowSeconds)),
+  );
+  if (window <= 0) return 0;
+
+  // FNV-1a 32-bit hash of the key -> a stable fraction in [0, 1).
+  let hash = 0x81_1C_9D_C5;
+  for (let i = 0; i < key.length; i++) {
+    hash ^= key.codePointAt(i) ?? 0;
+    hash = Math.imul(hash, 0x01_00_01_93);
+  }
+  const fraction = (hash >>> 0) / 0x1_00_00_00_00;
+  return Math.floor(fraction * window);
+}