@checkstack/healthcheck-backend 1.11.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@ import {
3
3
  setupHealthCheckWorker,
4
4
  scheduleHealthCheck,
5
5
  bootstrapHealthChecks,
6
+ recomputeSystemRollupHealth,
6
7
  type HealthCheckJobPayload,
7
8
  } from "./queue-executor";
8
9
  import type { HealthCheckCache } from "./cache";
@@ -792,7 +793,12 @@ describe("Queue-Based Health Check Executor", () => {
792
793
  collectorConfig?: Record<string, unknown>;
793
794
  /** Schema used to detect `x-templatable` fields for the render pass. */
794
795
  collectorConfigSchema?: z.ZodType<unknown>;
795
- }): Promise<Array<{ environment?: unknown; config?: unknown }>> {
796
+ }): Promise<{
797
+ /** Run-context captured per fanned-out run (env + rendered config). */
798
+ runs: Array<{ environment?: unknown; config?: unknown }>;
799
+ /** Payloads broadcast on `healthcheck.run.completed`, in order. */
800
+ runCompletedPayloads: Array<Record<string, unknown>>;
801
+ }> {
796
802
  const mockDb = createMockDb();
797
803
  const mockRegistry = createMockRegistry();
798
804
  const mockLogger = createMockLogger();
@@ -816,8 +822,13 @@ describe("Queue-Based Health Check Executor", () => {
816
822
  })),
817
823
  );
818
824
 
825
+ // The default full select chain (from().where(), groupBy, orderBy, ...)
826
+ // so the durable persist path (aggregate read + rollup) resolves instead
827
+ // of throwing on an unmodelled query shape - which is what lets the run
828
+ // reach the `HEALTH_CHECK_RUN_COMPLETED` broadcast.
829
+ const defaultSelect = mockDb.select;
819
830
  let selectCallCount = 0;
820
- (mockDb.select as any) = mock(() => {
831
+ (mockDb.select as any) = mock((...args: unknown[]) => {
821
832
  selectCallCount++;
822
833
  if (selectCallCount === 2) {
823
834
  return {
@@ -850,13 +861,7 @@ describe("Queue-Based Health Check Executor", () => {
850
861
  })),
851
862
  };
852
863
  }
853
- return {
854
- from: mock(() => ({
855
- innerJoin: mock(() => ({
856
- where: mock(() => Promise.resolve([])),
857
- })),
858
- })),
859
- };
864
+ return (defaultSelect as (...a: unknown[]) => unknown)(...args);
860
865
  });
861
866
 
862
867
  const captured: Array<{ environment?: unknown; config?: unknown }> = [];
@@ -940,11 +945,15 @@ describe("Queue-Based Health Check Executor", () => {
940
945
  }).catch(() => {});
941
946
  }
942
947
 
943
- return captured;
948
+ const runCompletedPayloads = mockSignalService
949
+ .getRecordedSignalsById("healthcheck.run.completed")
950
+ .map((r) => z.record(z.string(), z.unknown()).parse(r.payload));
951
+
952
+ return { runs: captured, runCompletedPayloads };
944
953
  }
945
954
 
946
955
  it("runs once per effective environment with that env in run-context (null selector = all)", async () => {
947
- const captured = await runFanOut({
956
+ const { runs: captured } = await runFanOut({
948
957
  environmentIds: null,
949
958
  membership: [
950
959
  { id: "prod", name: "Production", metadata: { baseUrl: "p" } },
@@ -965,8 +974,40 @@ describe("Queue-Based Health Check Executor", () => {
965
974
  });
966
975
  });
967
976
 
977
+ it("broadcasts the fanned-out environment on run.completed for each env", async () => {
978
+ const { runCompletedPayloads } = await runFanOut({
979
+ environmentIds: null,
980
+ membership: [
981
+ { id: "prod", name: "Production", metadata: { baseUrl: "p" } },
982
+ { id: "staging", name: "Staging", metadata: { baseUrl: "s" } },
983
+ ],
984
+ });
985
+
986
+ expect(runCompletedPayloads).toHaveLength(2);
987
+ expect(runCompletedPayloads[0]).toMatchObject({
988
+ environmentId: "prod",
989
+ environmentName: "Production",
990
+ });
991
+ expect(runCompletedPayloads[1]).toMatchObject({
992
+ environmentId: "staging",
993
+ environmentName: "Staging",
994
+ });
995
+ });
996
+
997
+ it("omits the environment on run.completed for an env-less run", async () => {
998
+ const { runCompletedPayloads } = await runFanOut({
999
+ environmentIds: [],
1000
+ membership: [{ id: "prod", name: "Production", metadata: {} }],
1001
+ });
1002
+
1003
+ expect(runCompletedPayloads).toHaveLength(1);
1004
+ // Zod optionals are omitted when unset, so env-less runs are unchanged.
1005
+ expect(runCompletedPayloads[0]?.environmentId).toBeUndefined();
1006
+ expect(runCompletedPayloads[0]?.environmentName).toBeUndefined();
1007
+ });
1008
+
968
1009
  it("renders x-templatable config fields per environment against environment.*", async () => {
969
- const captured = await runFanOut({
1010
+ const { runs: captured } = await runFanOut({
970
1011
  environmentIds: null,
971
1012
  membership: [
972
1013
  {
@@ -997,7 +1038,7 @@ describe("Queue-Based Health Check Executor", () => {
997
1038
  });
998
1039
 
999
1040
  it("renders environment.* to empty string for an env-less run (render-empty, §11.6)", async () => {
1000
- const captured = await runFanOut({
1041
+ const { runs: captured } = await runFanOut({
1001
1042
  environmentIds: [],
1002
1043
  membership: [
1003
1044
  { id: "prod", name: "Production", metadata: { baseUrl: "x" } },
@@ -1016,7 +1057,7 @@ describe("Queue-Based Health Check Executor", () => {
1016
1057
  });
1017
1058
 
1018
1059
  it("runs only the explicit subset, intersected with membership", async () => {
1019
- const captured = await runFanOut({
1060
+ const { runs: captured } = await runFanOut({
1020
1061
  environmentIds: ["staging"],
1021
1062
  membership: [
1022
1063
  { id: "prod", name: "Production", metadata: {} },
@@ -1029,7 +1070,7 @@ describe("Queue-Based Health Check Executor", () => {
1029
1070
  });
1030
1071
 
1031
1072
  it("runs exactly once with no environment when opting out ([] selector)", async () => {
1032
- const captured = await runFanOut({
1073
+ const { runs: captured } = await runFanOut({
1033
1074
  environmentIds: [],
1034
1075
  membership: [{ id: "prod", name: "Production", metadata: {} }],
1035
1076
  });
@@ -1039,7 +1080,7 @@ describe("Queue-Based Health Check Executor", () => {
1039
1080
  });
1040
1081
 
1041
1082
  it("runs exactly once env-less when the system has no environments (null selector, empty membership)", async () => {
1042
- const captured = await runFanOut({
1083
+ const { runs: captured } = await runFanOut({
1043
1084
  environmentIds: null,
1044
1085
  membership: [],
1045
1086
  });
@@ -1367,3 +1408,54 @@ describe("Queue-Based Health Check Executor", () => {
1367
1408
  });
1368
1409
  });
1369
1410
  });
1411
+
1412
+ describe("recomputeSystemRollupHealth", () => {
1413
+ /**
1414
+ * Verifies the pause/resume-driven rollup recompute:
1415
+ * - With no `getHealthEntity` bound, `writeHealthEntity` runs `apply`
1416
+ * directly, so the helper MUST call `service.getSystemHealthStatus` for
1417
+ * the bare `<systemId>` rollup id (no env qualifier).
1418
+ * - A `getSystemHealthStatus` failure must NOT propagate (the RPC must
1419
+ * survive a transient recompute error).
1420
+ */
1421
+ it("calls service.getSystemHealthStatus(systemId) for the rollup when no entity handle is bound", async () => {
1422
+ const getSystemHealthStatus = mock(async () => ({
1423
+ status: "healthy" as const,
1424
+ evaluatedAt: new Date(),
1425
+ checkStatuses: [],
1426
+ }));
1427
+ const service = { getSystemHealthStatus } as never;
1428
+
1429
+ await recomputeSystemRollupHealth({
1430
+ systemId: "sys-1",
1431
+ service,
1432
+ getHealthEntity: () => undefined,
1433
+ advisoryLock: mockAdvisoryLock,
1434
+ logger: createMockLogger(),
1435
+ });
1436
+
1437
+ expect(getSystemHealthStatus).toHaveBeenCalledTimes(1);
1438
+ expect(getSystemHealthStatus).toHaveBeenCalledWith("sys-1");
1439
+ });
1440
+
1441
+ it("swallows a recompute failure so the pause/resume RPC never throws", async () => {
1442
+ const getSystemHealthStatus = mock(async () => {
1443
+ throw new Error("db down");
1444
+ });
1445
+ const service = { getSystemHealthStatus } as never;
1446
+ const logger = createMockLogger();
1447
+
1448
+ await expect(
1449
+ recomputeSystemRollupHealth({
1450
+ systemId: "sys-1",
1451
+ service,
1452
+ getHealthEntity: () => undefined,
1453
+ advisoryLock: mockAdvisoryLock,
1454
+ logger,
1455
+ }),
1456
+ ).resolves.toBeUndefined();
1457
+
1458
+ // The error is logged for observability.
1459
+ expect(logger.error).toHaveBeenCalledTimes(1);
1460
+ });
1461
+ });
@@ -142,6 +142,7 @@ async function emitCheckCompletedHook({
142
142
  status,
143
143
  latencyMs,
144
144
  result,
145
+ environmentId,
145
146
  }: {
146
147
  getEmitHook: () => EmitHookFn | undefined;
147
148
  systemId: string;
@@ -149,6 +150,7 @@ async function emitCheckCompletedHook({
149
150
  status: string;
150
151
  latencyMs: number | undefined;
151
152
  result: Record<string, unknown> | undefined;
153
+ environmentId: string | null;
152
154
  }): Promise<void> {
153
155
  const emitHook = getEmitHook();
154
156
  if (!emitHook) return;
@@ -160,6 +162,7 @@ async function emitCheckCompletedHook({
160
162
  latencyMs,
161
163
  result,
162
164
  timestamp,
165
+ environmentId,
163
166
  });
164
167
  // Narrow follow-up — informational for automation triggers; the
165
168
  // auto-incident pipeline still runs on its own thresholds.
@@ -171,6 +174,7 @@ async function emitCheckCompletedHook({
171
174
  latencyMs,
172
175
  result,
173
176
  timestamp,
177
+ environmentId,
174
178
  });
175
179
  }
176
180
  }
@@ -235,6 +239,64 @@ export async function scheduleHealthCheck(props: {
235
239
  });
236
240
  }
237
241
 
242
+ /**
243
+ * Recompute and persist the SYSTEM ROLLUP `health` entity for `systemId`
244
+ * WITHOUT inserting a new run row.
245
+ *
246
+ * Used by configuration mutations that change which checks contribute to a
247
+ * system's aggregate WITHOUT producing a new run — today, `pause`/
248
+ * `resume`. Because `getSystemHealthStatus` excludes paused configs, the
249
+ * recomputed rollup may transition (e.g. `degraded → healthy` when the sole
250
+ * failing check is paused, or `healthy → degraded` when a check is resumed
251
+ * whose last in-window run was failing and the system had no other degraded
252
+ * checks). The framework diffs prev → next inside `handle.mutate` and emits a
253
+ * single `ENTITY_CHANGED` on a real transition, which the SLO engine's
254
+ * `onEntityChanged` handlers consume to close/open downtime events — so
255
+ * pausing a failing check closes its open SLO downtime, and resuming a
256
+ * still-failing check re-opens one on the next run.
257
+ *
258
+ * Mirrors the rollup write inside `executeHealthCheckJob` (no durable
259
+ * insert; just recompute + emit), serialized on the same per-entity
260
+ * `health:<systemId>` advisory lock so it can't race a concurrent run's
261
+ * rollup write. Best-effort: a reactivity failure is routed to `onError`
262
+ * and swallowed (the durable tables already hold the source-of-truth runs).
263
+ */
264
+ export async function recomputeSystemRollupHealth(args: {
265
+ systemId: string;
266
+ service: HealthCheckService;
267
+ getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
268
+ advisoryLock: AdvisoryLockService;
269
+ logger: Logger;
270
+ }): Promise<void> {
271
+ const { systemId, service, getHealthEntity, advisoryLock, logger } = args;
272
+ const rollupEntityId = encodeHealthEntityId({ systemId });
273
+ const makeHealthSerializer = createHealthEntitySerializer({ advisoryLock });
274
+ try {
275
+ await writeHealthEntity({
276
+ handle: getHealthEntity?.(),
277
+ entityId: rollupEntityId,
278
+ apply: async () => {
279
+ const rollupState = await service.getSystemHealthStatus(systemId);
280
+ return toHealthEntityView(rollupState);
281
+ },
282
+ serialize: makeHealthSerializer(rollupEntityId),
283
+ onError: (error) =>
284
+ logger.warn(
285
+ `Failed to mirror rollup health entity for ${systemId} (recompute)`,
286
+ error,
287
+ ),
288
+ });
289
+ } catch (error) {
290
+ // A recompute failure must never break the pause/resume RPC. The
291
+ // durable tables still hold the authoritative runs; the next run tick
292
+ // or the SLO self-heal (`reconcileOrphanedDowntime`) will converge.
293
+ logger.error(
294
+ `Failed to recompute system rollup health for ${systemId}`,
295
+ error,
296
+ );
297
+ }
298
+ }
299
+
238
300
  // Flapping detection no longer lives here. It moved into the automation
239
301
  // engine as a windowed-count gate on the `healthcheck.system_health_changed`
240
302
  // trigger (raw aggregated-health change + `filter` +
@@ -263,6 +325,21 @@ async function notifyStateChange(props: {
263
325
  configurationId: string;
264
326
  previousStatus: HealthCheckStatus;
265
327
  newStatus: HealthCheckStatus;
328
+ /**
329
+ * The environment this transition is scoped to. `null` (or `undefined`)
330
+ * means the rollout transition (the system rollup). A concrete string means
331
+ * the per-env slice — the body and collapse key are env-qualified so two
332
+ * failing envs don't merge into one card (see the changeset "Make
333
+ * healthcheck triggers env-scoped").
334
+ */
335
+ environmentId?: string | null;
336
+ /**
337
+ * Human-readable env name, included in the title/body. Best-effort: when
338
+ * absent (e.g. catastrophic job failure before env resolution) the message
339
+ * falls back to the bare system name. Resolution happens before the call
340
+ * site, so no extra catalog RPC here.
341
+ */
342
+ environmentName?: string;
266
343
  service: HealthCheckService;
267
344
  catalogClient: CatalogClient;
268
345
  notificationClient: NotificationClient;
@@ -276,6 +353,8 @@ async function notifyStateChange(props: {
276
353
  configurationId,
277
354
  previousStatus,
278
355
  newStatus,
356
+ environmentId,
357
+ environmentName,
279
358
  service,
280
359
  catalogClient,
281
360
  notificationClient,
@@ -284,6 +363,9 @@ async function notifyStateChange(props: {
284
363
  logger,
285
364
  } = props;
286
365
 
366
+ const envScoped = typeof environmentId === "string";
367
+ const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
368
+
287
369
  const transition = classifyTransition(previousStatus, newStatus);
288
370
  if (transition === "none") {
289
371
  return;
@@ -353,19 +435,23 @@ async function notifyStateChange(props: {
353
435
  let importance: "info" | "warning" | "critical";
354
436
 
355
437
  if (transition === "recovery") {
356
- title = `System health restored: ${systemName}`;
357
- body =
358
- `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
438
+ title = `System health restored${envSuffix}: ${systemName}`;
439
+ body = envScoped
440
+ ? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
441
+ : `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
359
442
  importance = "info";
360
443
  } else if (newStatus === "unhealthy") {
361
- title = `System health critical: ${systemName}`;
362
- body = `Health checks indicate **${systemName}** is unhealthy and may be down.`;
444
+ title = `System health critical${envSuffix}: ${systemName}`;
445
+ body = envScoped
446
+ ? `Health checks indicate **${systemName}** is unhealthy in environment **${environmentName ?? environmentId}** and may be down in that environment.`
447
+ : `Health checks indicate **${systemName}** is unhealthy and may be down.`;
363
448
  importance = "critical";
364
449
  } else {
365
450
  // degraded — either an escalation from healthy or a partial recovery
366
- title = `System health degraded: ${systemName}`;
367
- body =
368
- `Some health checks for **${systemName}** are failing. The system may be experiencing issues.`;
451
+ title = `System health degraded${envSuffix}: ${systemName}`;
452
+ body = envScoped
453
+ ? `Some health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are failing. That environment may be experiencing issues.`
454
+ : `Some health checks for **${systemName}** are failing. The system may be experiencing issues.`;
369
455
  importance = "warning";
370
456
  }
371
457
 
@@ -391,7 +477,14 @@ async function notifyStateChange(props: {
391
477
  body,
392
478
  importance,
393
479
  action: { label: actionLabel, url: actionUrl },
394
- collapseKey: systemHealthCollapseKey(systemId),
480
+ // Env-qualified collapse key so two failing envs of one system generate
481
+ // two independent notification cards (one per env) instead of merging
482
+ // -> operators see all env outages. The system-rollup transition
483
+ // (`environmentId === null`/undefined) keys on the bare systemId and
484
+ // therefore reuses the pre-existing single-card identity.
485
+ collapseKey: envScoped
486
+ ? systemHealthCollapseKey(systemId, environmentId)
487
+ : systemHealthCollapseKey(systemId),
395
488
  subjects: [
396
489
  createSystemSubject({
397
490
  id: systemId,
@@ -992,6 +1085,10 @@ async function executeHealthCheckJob(props: {
992
1085
  configurationName: configRow.configName,
993
1086
  status: result.status,
994
1087
  latencyMs: result.latencyMs,
1088
+ // Env-scoped fan-out: `environment` is null for the env-less run, so
1089
+ // `?.` yields undefined and those runs broadcast exactly as before.
1090
+ environmentId: environment?.id,
1091
+ environmentName: environment?.name,
995
1092
  });
996
1093
 
997
1094
  if (newState.status !== previousStatus) {
@@ -1013,6 +1110,8 @@ async function executeHealthCheckJob(props: {
1013
1110
  configurationId: configId,
1014
1111
  previousStatus,
1015
1112
  newStatus: newState.status,
1113
+ environmentId,
1114
+ environmentName: environment?.name,
1016
1115
  service,
1017
1116
  catalogClient,
1018
1117
  maintenanceClient,
@@ -1122,6 +1221,10 @@ async function executeHealthCheckJob(props: {
1122
1221
  configurationName: configRow.configName,
1123
1222
  status: result.status,
1124
1223
  latencyMs: result.latencyMs,
1224
+ // Env-scoped fan-out: `environment` is null for the env-less run, so
1225
+ // `?.` yields undefined and those runs broadcast exactly as before.
1226
+ environmentId: environment?.id,
1227
+ environmentName: environment?.name,
1125
1228
  });
1126
1229
 
1127
1230
  await emitCheckCompletedHook({
@@ -1131,6 +1234,7 @@ async function executeHealthCheckJob(props: {
1131
1234
  status: result.status,
1132
1235
  latencyMs: result.latencyMs,
1133
1236
  result: (result.metadata?.collectors as Record<string, unknown>) ?? undefined,
1237
+ environmentId,
1134
1238
  });
1135
1239
 
1136
1240
  if (newState.status !== previousStatus) {
@@ -1152,6 +1256,8 @@ async function executeHealthCheckJob(props: {
1152
1256
  configurationId: configId,
1153
1257
  previousStatus,
1154
1258
  newStatus: newState.status,
1259
+ environmentId,
1260
+ environmentName: environment?.name,
1155
1261
  service,
1156
1262
  catalogClient,
1157
1263
  maintenanceClient,
@@ -1357,6 +1463,7 @@ async function executeHealthCheckJob(props: {
1357
1463
  status: "unhealthy",
1358
1464
  latencyMs: undefined,
1359
1465
  result: undefined,
1466
+ environmentId: null,
1360
1467
  });
1361
1468
 
1362
1469
  if (newState.status !== previousStatus) {
@@ -0,0 +1,142 @@
1
+ import { describe, it, expect, mock, beforeEach } from "bun:test";
2
+ import { createHealthCheckRouter } from "./router";
3
+ import { createMockRpcContext } from "@checkstack/backend-api";
4
+ import { call } from "@orpc/server";
5
+ import type { HealthCheckCache } from "./cache";
6
+
7
+ /**
8
+ * Router-level tests for the pause/resume handlers' rollup-health recompute
9
+ * wiring. Pausing a configuration can flip the system's aggregate (e.g.
10
+ * degraded → healthy when the sole failing check is paused); the router
11
+ * MUST recompute the rollup `health` entity for every affected system so
12
+ * that transition reaches the SLO engine and closes any open downtime
13
+ * event. Resuming intentionally does NOT recompute — the next run drives
14
+ * any degraded transition (see the resume handler comment for rationale).
15
+ */
16
+
17
+ const passthroughCache: HealthCheckCache = {
18
+ wrapSystemHealthStatus: (_systemId, loader) => loader(),
19
+ invalidateSystem: async () => {},
20
+ invalidateAllSystems: async () => 0,
21
+ scope: {} as HealthCheckCache["scope"],
22
+ };
23
+
24
+ const mockUser = {
25
+ type: "user" as const,
26
+ id: "test-user",
27
+ accessRules: ["*"],
28
+ roles: ["admin"],
29
+ };
30
+
31
+ const CONFIG_ID = "cfg-1";
32
+
33
+ /**
34
+ * Mock db supporting the two query shapes the pause/resume handlers use:
35
+ * - `update().set().where()` for the paused flag flip
36
+ * - `select({systemId}).from(systemHealthChecks).where()` for
37
+ * `getSystemIdsForConfiguration`
38
+ *
39
+ * `systemIdsForConfig` is the configurable return for the SELECT query,
40
+ * letting each test script the affected-system set.
41
+ */
42
+ function createMockDb(systemIdsForConfig: string[]) {
43
+ const selectResult = Promise.resolve(
44
+ systemIdsForConfig.map((systemId) => ({ systemId })),
45
+ );
46
+ const whereMock = mock(() => selectResult);
47
+ const fromResult = Object.assign(Promise.resolve([]), {
48
+ where: whereMock,
49
+ });
50
+ const updateWhereMock = mock(() => Promise.resolve());
51
+ const updateSetMock = mock(() => ({ where: updateWhereMock }));
52
+ return {
53
+ select: mock(() => ({ from: mock(() => fromResult) })),
54
+ update: mock(() => ({ set: updateSetMock })),
55
+ insert: mock(() => ({
56
+ values: mock(() => ({
57
+ onConflictDoUpdate: mock(() => Promise.resolve()),
58
+ onConflictDoNothing: mock(() => Promise.resolve()),
59
+ returning: mock(() => Promise.resolve([])),
60
+ })),
61
+ })),
62
+ delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
63
+ execute: mock(() => Promise.resolve()),
64
+ };
65
+ }
66
+
67
+ function buildRouter(
68
+ systemIdsForConfig: string[],
69
+ recomputeCalls: string[],
70
+ ) {
71
+ const recomputeSystemRollupHealth = (systemId: string) => {
72
+ recomputeCalls.push(systemId);
73
+ return Promise.resolve();
74
+ };
75
+ return {
76
+ router: createHealthCheckRouter({
77
+ database: createMockDb(systemIdsForConfig) as never,
78
+ registry: { getStrategy: mock(() => undefined) } as never,
79
+ collectorRegistry: { getCollector: mock(() => undefined) } as never,
80
+ gitOpsClient: {
81
+ getProvenance: mock(() => Promise.resolve(null)),
82
+ } as never,
83
+ signalService: {
84
+ broadcast: mock(() => Promise.resolve()),
85
+ } as never,
86
+ getEmitHook: () => undefined,
87
+ cache: passthroughCache,
88
+ configService: {
89
+ get: mock(async () => undefined),
90
+ set: mock(async () => {}),
91
+ } as never,
92
+ catalogClient: { getSystem: mock(async () => null) } as never,
93
+ maintenanceClient: {
94
+ hasActiveMaintenance: mock(async () => ({ active: false })),
95
+ } as never,
96
+ logger: {
97
+ debug: mock(() => {}),
98
+ info: mock(() => {}),
99
+ warn: mock(() => {}),
100
+ error: mock(() => {}),
101
+ } as never,
102
+ recomputeSystemRollupHealth,
103
+ }),
104
+ };
105
+ }
106
+
107
+ describe("pauseConfiguration - rollup recompute wiring", () => {
108
+ it("recomputes the rollup health entity for every affected system", async () => {
109
+ const systemIds = ["sys-1", "sys-2", "sys-3"];
110
+ const recomputeCalls: string[] = [];
111
+ const { router } = buildRouter(systemIds, recomputeCalls);
112
+ const context = createMockRpcContext({ user: mockUser });
113
+
114
+ await call(router.pauseConfiguration, { id: CONFIG_ID }, { context });
115
+
116
+ expect(recomputeCalls).toHaveLength(3);
117
+ expect(recomputeCalls.sort()).toEqual(["sys-1", "sys-2", "sys-3"]);
118
+ });
119
+
120
+ it("still succeeds when no systems are assigned (no recompute calls)", async () => {
121
+ const recomputeCalls: string[] = [];
122
+ const { router } = buildRouter([], recomputeCalls);
123
+ const context = createMockRpcContext({ user: mockUser });
124
+
125
+ await call(router.pauseConfiguration, { id: CONFIG_ID }, { context });
126
+
127
+ expect(recomputeCalls).toHaveLength(0);
128
+ });
129
+ });
130
+
131
+ describe("resumeConfiguration - no recompute", () => {
132
+ it("does NOT recompute the rollup on resume (lazy: next run drives transition)", async () => {
133
+ const systemIds = ["sys-1", "sys-2"];
134
+ const recomputeCalls: string[] = [];
135
+ const { router } = buildRouter(systemIds, recomputeCalls);
136
+ const context = createMockRpcContext({ user: mockUser });
137
+
138
+ await call(router.resumeConfiguration, { id: CONFIG_ID }, { context });
139
+
140
+ expect(recomputeCalls).toHaveLength(0);
141
+ });
142
+ });
package/src/router.ts CHANGED
@@ -57,6 +57,16 @@ export const createHealthCheckRouter = (opts: {
57
57
  * Optional so existing tests can omit it.
58
58
  */
59
59
  signalService?: SignalService;
60
+ /**
61
+ * Recompute and persist the system rollup `health` entity for a single
62
+ * system, used by `pauseConfiguration` to drive the SLO engine when the
63
+ * recomputed aggregate transitions (e.g. degraded → healthy when the
64
+ * sole failing check is paused, which closes the open SLO downtime
65
+ * event via the `HEALTH_ENTITY_KIND` "recovered" transition). Optional so
66
+ * tests can omit it; when absent, pause just flips the flag and the next
67
+ * run / the SLO self-heal converge the rollup lazily.
68
+ */
69
+ recomputeSystemRollupHealth?: (systemId: string) => Promise<void>;
60
70
  }) => {
61
71
  const {
62
72
  database,
@@ -69,6 +79,7 @@ export const createHealthCheckRouter = (opts: {
69
79
  maintenanceClient,
70
80
  logger,
71
81
  signalService,
82
+ recomputeSystemRollupHealth,
72
83
  } = opts;
73
84
  // Create service instance once - shared across all handlers
74
85
  const service = new HealthCheckService(
@@ -312,6 +323,31 @@ export const createHealthCheckRouter = (opts: {
312
323
  action: "updated",
313
324
  configurationId: input.id,
314
325
  });
326
+
327
+ // Recompute the rollup `health` entity for every system this config
328
+ // is assigned to (enabled assignments only). Because
329
+ // `getSystemHealthStatus` now excludes paused configs, the recomputed
330
+ // rollup may transition degraded → healthy, which emits the
331
+ // `HEALTH_ENTITY_KIND` "recovered" edge the SLO engine consumes to
332
+ // close any open downtime event attributed to this check. If the
333
+ // system stays degraded (other failing checks), no edge fires and
334
+ // the open event correctly persists. Best-effort: a recompute
335
+ // failure is logged inside the helper and never breaks the RPC.
336
+ if (recomputeSystemRollupHealth) {
337
+ try {
338
+ const systemIds =
339
+ await service.getSystemIdsForConfiguration(input.id);
340
+ await Promise.all(
341
+ systemIds.map((systemId) =>
342
+ recomputeSystemRollupHealth(systemId),
343
+ ),
344
+ );
345
+ } catch (error) {
346
+ logger.warn(
347
+ `Failed to recompute rollup health after pausing config ${input.id}: ${extractErrorMessage(error, "unknown")}`,
348
+ );
349
+ }
350
+ }
315
351
  }),
316
352
 
317
353
  resumeConfiguration: os.resumeConfiguration.handler(async ({ input }) => {
@@ -323,6 +359,17 @@ export const createHealthCheckRouter = (opts: {
323
359
  action: "updated",
324
360
  configurationId: input.id,
325
361
  });
362
+ // Intentionally NO rollup recompute on resume. The check's last
363
+ // in-window run may have been failing, but we don't know whether the
364
+ // underlying condition is still present — only a fresh run can tell.
365
+ // Recomputing now would open a new SLO downtime event based on stale
366
+ // data and then potentially close it on the next successful run,
367
+ // fabricating a brief false-positive downtime. Instead, let the
368
+ // recurring job's next tick drive any degraded transition: if the
369
+ // check still fails, the new unhealthy run recomputes the rollup and
370
+ // the SLO engine opens a fresh downtime event (the previous event was
371
+ // closed on pause, so the idempotent guard in `handleSystemDown`
372
+ // doesn't suppress it). If the check now passes, no event opens.
326
373
  }),
327
374
 
328
375
  getSystemConfigurations: os.getSystemConfigurations.handler(