@checkstack/healthcheck-backend 1.11.1 → 1.13.0

package/src/queue-executor.test.ts

@@ -3,6 +3,7 @@ import {
   setupHealthCheckWorker,
   scheduleHealthCheck,
   bootstrapHealthChecks,
+  recomputeSystemRollupHealth,
   type HealthCheckJobPayload,
 } from "./queue-executor";
 import type { HealthCheckCache } from "./cache";
@@ -792,7 +793,12 @@ describe("Queue-Based Health Check Executor", () => {
       collectorConfig?: Record<string, unknown>;
       /** Schema used to detect `x-templatable` fields for the render pass. */
       collectorConfigSchema?: z.ZodType<unknown>;
-    }): Promise<Array<{ environment?: unknown; config?: unknown }>> {
+    }): Promise<{
+      /** Run-context captured per fanned-out run (env + rendered config). */
+      runs: Array<{ environment?: unknown; config?: unknown }>;
+      /** Payloads broadcast on `healthcheck.run.completed`, in order. */
+      runCompletedPayloads: Array<Record<string, unknown>>;
+    }> {
       const mockDb = createMockDb();
       const mockRegistry = createMockRegistry();
       const mockLogger = createMockLogger();
@@ -816,8 +822,13 @@ describe("Queue-Based Health Check Executor", () => {
         })),
       );
 
+      // The default full select chain (from().where(), groupBy, orderBy, ...)
+      // so the durable persist path (aggregate read + rollup) resolves instead
+      // of throwing on an unmodelled query shape - which is what lets the run
+      // reach the `HEALTH_CHECK_RUN_COMPLETED` broadcast.
+      const defaultSelect = mockDb.select;
       let selectCallCount = 0;
-      (mockDb.select as any) = mock(() => {
+      (mockDb.select as any) = mock((...args: unknown[]) => {
         selectCallCount++;
         if (selectCallCount === 2) {
           return {
@@ -850,13 +861,7 @@ describe("Queue-Based Health Check Executor", () => {
             })),
           };
         }
-        return {
-          from: mock(() => ({
-            innerJoin: mock(() => ({
-              where: mock(() => Promise.resolve([])),
-            })),
-          })),
-        };
+        return (defaultSelect as (...a: unknown[]) => unknown)(...args);
       });
 
       const captured: Array<{ environment?: unknown; config?: unknown }> = [];
@@ -940,11 +945,15 @@ describe("Queue-Based Health Check Executor", () => {
         }).catch(() => {});
       }
 
-      return captured;
+      const runCompletedPayloads = mockSignalService
+        .getRecordedSignalsById("healthcheck.run.completed")
+        .map((r) => z.record(z.string(), z.unknown()).parse(r.payload));
+
+      return { runs: captured, runCompletedPayloads };
     }
 
     it("runs once per effective environment with that env in run-context (null selector = all)", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: null,
         membership: [
           { id: "prod", name: "Production", metadata: { baseUrl: "p" } },
@@ -965,8 +974,40 @@ describe("Queue-Based Health Check Executor", () => {
       });
     });
 
+    it("broadcasts the fanned-out environment on run.completed for each env", async () => {
+      const { runCompletedPayloads } = await runFanOut({
+        environmentIds: null,
+        membership: [
+          { id: "prod", name: "Production", metadata: { baseUrl: "p" } },
+          { id: "staging", name: "Staging", metadata: { baseUrl: "s" } },
+        ],
+      });
+
+      expect(runCompletedPayloads).toHaveLength(2);
+      expect(runCompletedPayloads[0]).toMatchObject({
+        environmentId: "prod",
+        environmentName: "Production",
+      });
+      expect(runCompletedPayloads[1]).toMatchObject({
+        environmentId: "staging",
+        environmentName: "Staging",
+      });
+    });
+
+    it("omits the environment on run.completed for an env-less run", async () => {
+      const { runCompletedPayloads } = await runFanOut({
+        environmentIds: [],
+        membership: [{ id: "prod", name: "Production", metadata: {} }],
+      });
+
+      expect(runCompletedPayloads).toHaveLength(1);
+      // Zod optionals are omitted when unset, so env-less runs are unchanged.
+      expect(runCompletedPayloads[0]?.environmentId).toBeUndefined();
+      expect(runCompletedPayloads[0]?.environmentName).toBeUndefined();
+    });
+
     it("renders x-templatable config fields per environment against environment.*", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: null,
         membership: [
           {
@@ -997,7 +1038,7 @@ describe("Queue-Based Health Check Executor", () => {
     });
 
     it("renders environment.* to empty string for an env-less run (render-empty, §11.6)", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: [],
         membership: [
           { id: "prod", name: "Production", metadata: { baseUrl: "x" } },
@@ -1016,7 +1057,7 @@ describe("Queue-Based Health Check Executor", () => {
     });
 
     it("runs only the explicit subset, intersected with membership", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: ["staging"],
         membership: [
           { id: "prod", name: "Production", metadata: {} },
@@ -1029,7 +1070,7 @@ describe("Queue-Based Health Check Executor", () => {
     });
 
     it("runs exactly once with no environment when opting out ([] selector)", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: [],
         membership: [{ id: "prod", name: "Production", metadata: {} }],
       });
@@ -1039,7 +1080,7 @@ describe("Queue-Based Health Check Executor", () => {
     });
 
     it("runs exactly once env-less when the system has no environments (null selector, empty membership)", async () => {
-      const captured = await runFanOut({
+      const { runs: captured } = await runFanOut({
         environmentIds: null,
         membership: [],
       });
@@ -1367,3 +1408,54 @@ describe("Queue-Based Health Check Executor", () => {
     });
   });
 });
+
+describe("recomputeSystemRollupHealth", () => {
+  /**
+   * Verifies the pause/resume-driven rollup recompute:
+   *  - With no `getHealthEntity` bound, `writeHealthEntity` runs `apply`
+   *    directly, so the helper MUST call `service.getSystemHealthStatus` for
+   *    the bare `<systemId>` rollup id (no env qualifier).
+   *  - A `getSystemHealthStatus` failure must NOT propagate (the RPC must
+   *    survive a transient recompute error).
+   */
+  it("calls service.getSystemHealthStatus(systemId) for the rollup when no entity handle is bound", async () => {
+    const getSystemHealthStatus = mock(async () => ({
+      status: "healthy" as const,
+      evaluatedAt: new Date(),
+      checkStatuses: [],
+    }));
+    const service = { getSystemHealthStatus } as never;
+
+    await recomputeSystemRollupHealth({
+      systemId: "sys-1",
+      service,
+      getHealthEntity: () => undefined,
+      advisoryLock: mockAdvisoryLock,
+      logger: createMockLogger(),
+    });
+
+    expect(getSystemHealthStatus).toHaveBeenCalledTimes(1);
+    expect(getSystemHealthStatus).toHaveBeenCalledWith("sys-1");
+  });
+
+  it("swallows a recompute failure so the pause/resume RPC never throws", async () => {
+    const getSystemHealthStatus = mock(async () => {
+      throw new Error("db down");
+    });
+    const service = { getSystemHealthStatus } as never;
+    const logger = createMockLogger();
+
+    await expect(
+      recomputeSystemRollupHealth({
+        systemId: "sys-1",
+        service,
+        getHealthEntity: () => undefined,
+        advisoryLock: mockAdvisoryLock,
+        logger,
+      }),
+    ).resolves.toBeUndefined();
+
+    // The error is logged for observability.
+    expect(logger.error).toHaveBeenCalledTimes(1);
+  });
+});

package/src/queue-executor.ts

@@ -142,6 +142,7 @@ async function emitCheckCompletedHook({
   status,
   latencyMs,
   result,
+  environmentId,
 }: {
   getEmitHook: () => EmitHookFn | undefined;
   systemId: string;
@@ -149,6 +150,7 @@ async function emitCheckCompletedHook({
   status: string;
   latencyMs: number | undefined;
   result: Record<string, unknown> | undefined;
+  environmentId: string | null;
 }): Promise<void> {
   const emitHook = getEmitHook();
   if (!emitHook) return;
@@ -160,6 +162,7 @@ async function emitCheckCompletedHook({
     latencyMs,
     result,
     timestamp,
+    environmentId,
   });
   // Narrow follow-up — informational for automation triggers; the
   // auto-incident pipeline still runs on its own thresholds.
@@ -171,6 +174,7 @@ async function emitCheckCompletedHook({
       latencyMs,
       result,
       timestamp,
+      environmentId,
     });
   }
 }
@@ -235,6 +239,64 @@ export async function scheduleHealthCheck(props: {
   });
 }
 
+/**
+ * Recompute and persist the SYSTEM ROLLUP `health` entity for `systemId`
+ * WITHOUT inserting a new run row.
+ *
+ * Used by configuration mutations that change which checks contribute to a
+ * system's aggregate WITHOUT producing a new run — today, `pause`/
+ * `resume`. Because `getSystemHealthStatus` excludes paused configs, the
+ * recomputed rollup may transition (e.g. `degraded → healthy` when the sole
+ * failing check is paused, or `healthy → degraded` when a check is resumed
+ * whose last in-window run was failing and the system had no other degraded
+ * checks). The framework diffs prev → next inside `handle.mutate` and emits a
+ * single `ENTITY_CHANGED` on a real transition, which the SLO engine's
+ * `onEntityChanged` handlers consume to close/open downtime events — so
+ * pausing a failing check closes its open SLO downtime, and resuming a
+ * still-failing check re-opens one on the next run.
+ *
+ * Mirrors the rollup write inside `executeHealthCheckJob` (no durable
+ * insert; just recompute + emit), serialized on the same per-entity
+ * `health:<systemId>` advisory lock so it can't race a concurrent run's
+ * rollup write. Best-effort: a reactivity failure is routed to `onError`
+ * and swallowed (the durable tables already hold the source-of-truth runs).
+ */
+export async function recomputeSystemRollupHealth(args: {
+  systemId: string;
+  service: HealthCheckService;
+  getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
+  advisoryLock: AdvisoryLockService;
+  logger: Logger;
+}): Promise<void> {
+  const { systemId, service, getHealthEntity, advisoryLock, logger } = args;
+  const rollupEntityId = encodeHealthEntityId({ systemId });
+  const makeHealthSerializer = createHealthEntitySerializer({ advisoryLock });
+  try {
+    await writeHealthEntity({
+      handle: getHealthEntity?.(),
+      entityId: rollupEntityId,
+      apply: async () => {
+        const rollupState = await service.getSystemHealthStatus(systemId);
+        return toHealthEntityView(rollupState);
+      },
+      serialize: makeHealthSerializer(rollupEntityId),
+      onError: (error) =>
+        logger.warn(
+          `Failed to mirror rollup health entity for ${systemId} (recompute)`,
+          error,
+        ),
+    });
+  } catch (error) {
+    // A recompute failure must never break the pause/resume RPC. The
+    // durable tables still hold the authoritative runs; the next run tick
+    // or the SLO self-heal (`reconcileOrphanedDowntime`) will converge.
+    logger.error(
+      `Failed to recompute system rollup health for ${systemId}`,
+      error,
+    );
+  }
+}
+
 // Flapping detection no longer lives here. It moved into the automation
 // engine as a windowed-count gate on the `healthcheck.system_health_changed`
 // trigger (raw aggregated-health change + `filter` +
@@ -263,6 +325,21 @@ async function notifyStateChange(props: {
   configurationId: string;
   previousStatus: HealthCheckStatus;
   newStatus: HealthCheckStatus;
+  /**
+   * The environment this transition is scoped to. `null` (or `undefined`)
+   * means the rollout transition (the system rollup). A concrete string means
+   * the per-env slice — the body and collapse key are env-qualified so two
+   * failing envs don't merge into one card (see the changeset "Make
+   * healthcheck triggers env-scoped").
+   */
+  environmentId?: string | null;
+  /**
+   * Human-readable env name, included in the title/body. Best-effort: when
+   * absent (e.g. catastrophic job failure before env resolution) the message
+   * falls back to the bare system name. Resolution happens before the call
+   * site, so no extra catalog RPC here.
+   */
+  environmentName?: string;
   service: HealthCheckService;
   catalogClient: CatalogClient;
   notificationClient: NotificationClient;
@@ -276,6 +353,8 @@ async function notifyStateChange(props: {
     configurationId,
     previousStatus,
     newStatus,
+    environmentId,
+    environmentName,
     service,
     catalogClient,
     notificationClient,
@@ -284,6 +363,9 @@ async function notifyStateChange(props: {
     logger,
   } = props;
 
+  const envScoped = typeof environmentId === "string";
+  const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
+
   const transition = classifyTransition(previousStatus, newStatus);
   if (transition === "none") {
     return;
@@ -353,19 +435,23 @@ async function notifyStateChange(props: {
   let importance: "info" | "warning" | "critical";
 
   if (transition === "recovery") {
-    title = `System health restored: ${systemName}`;
-    body =
-      `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
+    title = `System health restored${envSuffix}: ${systemName}`;
+    body = envScoped
+      ? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
+      : `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
     importance = "info";
   } else if (newStatus === "unhealthy") {
-    title = `System health critical: ${systemName}`;
-    body = `Health checks indicate **${systemName}** is unhealthy and may be down.`;
+    title = `System health critical${envSuffix}: ${systemName}`;
+    body = envScoped
+      ? `Health checks indicate **${systemName}** is unhealthy in environment **${environmentName ?? environmentId}** and may be down in that environment.`
+      : `Health checks indicate **${systemName}** is unhealthy and may be down.`;
     importance = "critical";
   } else {
     // degraded — either an escalation from healthy or a partial recovery
-    title = `System health degraded: ${systemName}`;
-    body =
-      `Some health checks for **${systemName}** are failing. The system may be experiencing issues.`;
+    title = `System health degraded${envSuffix}: ${systemName}`;
+    body = envScoped
+      ? `Some health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are failing. That environment may be experiencing issues.`
+      : `Some health checks for **${systemName}** are failing. The system may be experiencing issues.`;
     importance = "warning";
   }
 
@@ -391,7 +477,14 @@ async function notifyStateChange(props: {
       body,
       importance,
       action: { label: actionLabel, url: actionUrl },
-      collapseKey: systemHealthCollapseKey(systemId),
+      // Env-qualified collapse key so two failing envs of one system generate
+      // two independent notification cards (one per env) instead of merging
+      // -> operators see all env outages. The system-rollup transition
+      // (`environmentId === null`/undefined) keys on the bare systemId and
+      // therefore reuses the pre-existing single-card identity.
+      collapseKey: envScoped
+        ? systemHealthCollapseKey(systemId, environmentId)
+        : systemHealthCollapseKey(systemId),
       subjects: [
         createSystemSubject({
           id: systemId,
@@ -992,6 +1085,10 @@ async function executeHealthCheckJob(props: {
         configurationName: configRow.configName,
         status: result.status,
         latencyMs: result.latencyMs,
+        // Env-scoped fan-out: `environment` is null for the env-less run, so
+        // `?.` yields undefined and those runs broadcast exactly as before.
+        environmentId: environment?.id,
+        environmentName: environment?.name,
       });
 
       if (newState.status !== previousStatus) {
@@ -1013,6 +1110,8 @@ async function executeHealthCheckJob(props: {
           configurationId: configId,
           previousStatus,
           newStatus: newState.status,
+          environmentId,
+          environmentName: environment?.name,
           service,
           catalogClient,
           maintenanceClient,
@@ -1122,6 +1221,10 @@ async function executeHealthCheckJob(props: {
       configurationName: configRow.configName,
       status: result.status,
       latencyMs: result.latencyMs,
+      // Env-scoped fan-out: `environment` is null for the env-less run, so
+      // `?.` yields undefined and those runs broadcast exactly as before.
+      environmentId: environment?.id,
+      environmentName: environment?.name,
     });
 
     await emitCheckCompletedHook({
@@ -1131,6 +1234,7 @@ async function executeHealthCheckJob(props: {
       status: result.status,
       latencyMs: result.latencyMs,
       result: (result.metadata?.collectors as Record<string, unknown>) ?? undefined,
+      environmentId,
     });
 
     if (newState.status !== previousStatus) {
@@ -1152,6 +1256,8 @@ async function executeHealthCheckJob(props: {
         configurationId: configId,
         previousStatus,
         newStatus: newState.status,
+        environmentId,
+        environmentName: environment?.name,
         service,
         catalogClient,
         maintenanceClient,
@@ -1357,6 +1463,7 @@ async function executeHealthCheckJob(props: {
       status: "unhealthy",
       latencyMs: undefined,
       result: undefined,
+      environmentId: null,
     });
 
     if (newState.status !== previousStatus) {

package/src/router-pause-recompute.test.ts

@@ -0,0 +1,142 @@
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+import { createHealthCheckRouter } from "./router";
+import { createMockRpcContext } from "@checkstack/backend-api";
+import { call } from "@orpc/server";
+import type { HealthCheckCache } from "./cache";
+
+/**
+ * Router-level tests for the pause/resume handlers' rollup-health recompute
+ * wiring. Pausing a configuration can flip the system's aggregate (e.g.
+ * degraded → healthy when the sole failing check is paused); the router
+ * MUST recompute the rollup `health` entity for every affected system so
+ * that transition reaches the SLO engine and closes any open downtime
+ * event. Resuming intentionally does NOT recompute — the next run drives
+ * any degraded transition (see the resume handler comment for rationale).
+ */
+
+const passthroughCache: HealthCheckCache = {
+  wrapSystemHealthStatus: (_systemId, loader) => loader(),
+  invalidateSystem: async () => {},
+  invalidateAllSystems: async () => 0,
+  scope: {} as HealthCheckCache["scope"],
+};
+
+const mockUser = {
+  type: "user" as const,
+  id: "test-user",
+  accessRules: ["*"],
+  roles: ["admin"],
+};
+
+const CONFIG_ID = "cfg-1";
+
+/**
+ * Mock db supporting the two query shapes the pause/resume handlers use:
+ *  - `update().set().where()` for the paused flag flip
+ *  - `select({systemId}).from(systemHealthChecks).where()` for
+ *    `getSystemIdsForConfiguration`
+ *
+ * `systemIdsForConfig` is the configurable return for the SELECT query,
+ * letting each test script the affected-system set.
+ */
+function createMockDb(systemIdsForConfig: string[]) {
+  const selectResult = Promise.resolve(
+    systemIdsForConfig.map((systemId) => ({ systemId })),
+  );
+  const whereMock = mock(() => selectResult);
+  const fromResult = Object.assign(Promise.resolve([]), {
+    where: whereMock,
+  });
+  const updateWhereMock = mock(() => Promise.resolve());
+  const updateSetMock = mock(() => ({ where: updateWhereMock }));
+  return {
+    select: mock(() => ({ from: mock(() => fromResult) })),
+    update: mock(() => ({ set: updateSetMock })),
+    insert: mock(() => ({
+      values: mock(() => ({
+        onConflictDoUpdate: mock(() => Promise.resolve()),
+        onConflictDoNothing: mock(() => Promise.resolve()),
+        returning: mock(() => Promise.resolve([])),
+      })),
+    })),
+    delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
+    execute: mock(() => Promise.resolve()),
+  };
+}
+
+function buildRouter(
+  systemIdsForConfig: string[],
+  recomputeCalls: string[],
+) {
+  const recomputeSystemRollupHealth = (systemId: string) => {
+    recomputeCalls.push(systemId);
+    return Promise.resolve();
+  };
+  return {
+    router: createHealthCheckRouter({
+      database: createMockDb(systemIdsForConfig) as never,
+      registry: { getStrategy: mock(() => undefined) } as never,
+      collectorRegistry: { getCollector: mock(() => undefined) } as never,
+      gitOpsClient: {
+        getProvenance: mock(() => Promise.resolve(null)),
+      } as never,
+      signalService: {
+        broadcast: mock(() => Promise.resolve()),
+      } as never,
+      getEmitHook: () => undefined,
+      cache: passthroughCache,
+      configService: {
+        get: mock(async () => undefined),
+        set: mock(async () => {}),
+      } as never,
+      catalogClient: { getSystem: mock(async () => null) } as never,
+      maintenanceClient: {
+        hasActiveMaintenance: mock(async () => ({ active: false })),
+      } as never,
+      logger: {
+        debug: mock(() => {}),
+        info: mock(() => {}),
+        warn: mock(() => {}),
+        error: mock(() => {}),
+      } as never,
+      recomputeSystemRollupHealth,
+    }),
+  };
+}
+
+describe("pauseConfiguration - rollup recompute wiring", () => {
+  it("recomputes the rollup health entity for every affected system", async () => {
+    const systemIds = ["sys-1", "sys-2", "sys-3"];
+    const recomputeCalls: string[] = [];
+    const { router } = buildRouter(systemIds, recomputeCalls);
+    const context = createMockRpcContext({ user: mockUser });
+
+    await call(router.pauseConfiguration, { id: CONFIG_ID }, { context });
+
+    expect(recomputeCalls).toHaveLength(3);
+    expect(recomputeCalls.sort()).toEqual(["sys-1", "sys-2", "sys-3"]);
+  });
+
+  it("still succeeds when no systems are assigned (no recompute calls)", async () => {
+    const recomputeCalls: string[] = [];
+    const { router } = buildRouter([], recomputeCalls);
+    const context = createMockRpcContext({ user: mockUser });
+
+    await call(router.pauseConfiguration, { id: CONFIG_ID }, { context });
+
+    expect(recomputeCalls).toHaveLength(0);
+  });
+});
+
+describe("resumeConfiguration - no recompute", () => {
+  it("does NOT recompute the rollup on resume (lazy: next run drives transition)", async () => {
+    const systemIds = ["sys-1", "sys-2"];
+    const recomputeCalls: string[] = [];
+    const { router } = buildRouter(systemIds, recomputeCalls);
+    const context = createMockRpcContext({ user: mockUser });
+
+    await call(router.resumeConfiguration, { id: CONFIG_ID }, { context });
+
+    expect(recomputeCalls).toHaveLength(0);
+  });
+});