@checkstack/healthcheck-backend 1.17.0 → 1.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +559 -0
  2. package/package.json +32 -29
  3. package/src/adaptive-timeout.test.ts +91 -0
  4. package/src/adaptive-timeout.ts +75 -0
  5. package/src/ai/system-signals-contributor.test.ts +2 -0
  6. package/src/automations.test.ts +47 -0
  7. package/src/automations.ts +19 -3
  8. package/src/health-notification-content.test.ts +89 -0
  9. package/src/health-notification-content.ts +138 -0
  10. package/src/healthcheck-gitops-kinds.test.ts +34 -2
  11. package/src/healthcheck-gitops-kinds.ts +17 -13
  12. package/src/index.ts +58 -6
  13. package/src/migration-chain-contract.test.ts +7 -1
  14. package/src/notification-policy.test.ts +19 -0
  15. package/src/notification-policy.ts +26 -0
  16. package/src/queue-executor.test.ts +391 -338
  17. package/src/queue-executor.ts +426 -362
  18. package/src/realtime-aggregation.ts +9 -2
  19. package/src/rollup-consumer.test.ts +191 -0
  20. package/src/rollup-consumer.ts +160 -0
  21. package/src/router.ts +46 -13
  22. package/src/schedule-jitter.test.ts +69 -0
  23. package/src/schedule-jitter.ts +50 -0
  24. package/src/schedule-reconciler.it.test.ts +453 -0
  25. package/src/schedule-reconciler.test.ts +418 -0
  26. package/src/schedule-reconciler.ts +304 -0
  27. package/src/service-batching.test.ts +106 -0
  28. package/src/service-bulk-counts.it.test.ts +144 -0
  29. package/src/service-bulk-run-stats.it.test.ts +197 -0
  30. package/src/service-ordering.test.ts +10 -2
  31. package/src/service-paused-filter.test.ts +27 -7
  32. package/src/service-rollup-worst-wins.test.ts +221 -124
  33. package/src/service.ts +557 -266
  34. package/src/slow-check-admission.test.ts +184 -0
  35. package/src/slow-check-admission.ts +101 -0
  36. package/src/slow-check-classifier.test.ts +155 -0
  37. package/src/slow-check-classifier.ts +137 -0
  38. package/src/slow-check-config.ts +102 -0
  39. package/src/status-page/rollup.test.ts +40 -0
  40. package/src/status-page/rollup.ts +27 -0
  41. package/src/status-page/widgets.test.ts +303 -0
  42. package/src/status-page/widgets.ts +155 -39
  43. package/src/suspect-lane.test.ts +50 -0
  44. package/src/suspect-lane.ts +61 -0
@@ -0,0 +1,91 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import {
3
+ adaptiveTimeout,
4
+ DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
5
+ } from "./adaptive-timeout";
6
+
7
+ describe("adaptiveTimeout", () => {
8
+ const configuredMs = 30_000;
9
+
10
+ test("guardrail 1: no baseline => never shrink", () => {
11
+ expect(
12
+ adaptiveTimeout({
13
+ configuredMs,
14
+ healthyBaselineMs: undefined,
15
+ isSuspect: true,
16
+ isRecoveryProbe: false,
17
+ }),
18
+ ).toBe(configuredMs);
19
+ });
20
+
21
+ test("non-suspect check keeps the full configured timeout", () => {
22
+ expect(
23
+ adaptiveTimeout({
24
+ configuredMs,
25
+ healthyBaselineMs: 200,
26
+ isSuspect: false,
27
+ isRecoveryProbe: false,
28
+ }),
29
+ ).toBe(configuredMs);
30
+ });
31
+
32
+ test("guardrail 3: recovery probe always uses the full configured timeout", () => {
33
+ expect(
34
+ adaptiveTimeout({
35
+ configuredMs,
36
+ healthyBaselineMs: 200,
37
+ isSuspect: true,
38
+ isRecoveryProbe: true,
39
+ }),
40
+ ).toBe(configuredMs);
41
+ });
42
+
43
+ test("fast healthy check shrinks toward the floor (200ms => ~1s)", () => {
44
+ // 200 * 1.5 = 300 < floor => clamped to the absolute floor.
45
+ expect(
46
+ adaptiveTimeout({
47
+ configuredMs,
48
+ healthyBaselineMs: 200,
49
+ isSuspect: true,
50
+ isRecoveryProbe: false,
51
+ }),
52
+ ).toBe(DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS);
53
+ });
54
+
55
+ test("deadlock guard: a slow-but-healthy check (10s) never shrinks below its own latency", () => {
56
+ // 10_000 * 1.5 = 15_000: a recovering 10s run still passes at 15s.
57
+ const t = adaptiveTimeout({
58
+ configuredMs,
59
+ healthyBaselineMs: 10_000,
60
+ isSuspect: true,
61
+ isRecoveryProbe: false,
62
+ });
63
+ expect(t).toBe(15_000);
64
+ expect(t).toBeGreaterThan(10_000);
65
+ });
66
+
67
+ test("shrink never exceeds the configured timeout", () => {
68
+ // baseline*factor would exceed configured => clamp down to configured.
69
+ expect(
70
+ adaptiveTimeout({
71
+ configuredMs: 5_000,
72
+ healthyBaselineMs: 8_000,
73
+ isSuspect: true,
74
+ isRecoveryProbe: false,
75
+ }),
76
+ ).toBe(5_000);
77
+ });
78
+
79
+ test("honors custom safetyFactor and absoluteFloor", () => {
80
+ expect(
81
+ adaptiveTimeout({
82
+ configuredMs,
83
+ healthyBaselineMs: 1_000,
84
+ isSuspect: true,
85
+ isRecoveryProbe: false,
86
+ safetyFactor: 2,
87
+ absoluteFloorMs: 500,
88
+ }),
89
+ ).toBe(2_000);
90
+ });
91
+ });
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Adaptive execution timeout for slot-hogging health checks.
3
+ *
4
+ * A check that hangs holds its concurrency slot for the FULL configured timeout
5
+ * (see `queue-executor.ts` — the per-env `Promise.race` frees the slot only when
6
+ * the timeout fires). When a target is consistently timing out, we can free that
7
+ * slot sooner by probing it with a SHORTER timeout — but only down to a value at
8
+ * which a genuinely HEALTHY run of this same check would still succeed, or we
9
+ * would abort even a recovering target and never let it go healthy again.
10
+ *
11
+ * The floor is therefore derived from the check's OWN measured healthy latency,
12
+ * never a global constant. Four guardrails make a recovery deadlock impossible:
13
+ *
14
+ * 1. No baseline (no recent healthy run) => NO shrink. Without evidence of the
15
+ * target's healthy latency we cannot tell "hung" from "legitimately slow and
16
+ * about to succeed", so we keep the full configured timeout.
17
+ * 2. The baseline is computed from SUCCESSFUL runs only (caller's job), so a
18
+ * timed-out run (latency ~= timeout) can never ratchet the floor downward.
19
+ * 3. The periodic recovery probe passes `isRecoveryProbe: true` and always gets
20
+ * the FULL configured timeout — so a genuinely-slow-but-healthy target (e.g.
21
+ * a 10s computation) is always eventually re-measured at its real latency.
22
+ * This is the by-construction deadlock breaker.
23
+ * 4. A single healthy run clears the suspect classification upstream, so the
24
+ * full timeout is restored immediately (hysteresis).
25
+ */
26
+
27
+ /** Default multiplier applied to the healthy-latency baseline. */
28
+ export const DEFAULT_TIMEOUT_SAFETY_FACTOR = 1.5;
29
+ /** Default absolute lower bound; the timeout is never shrunk below this. */
30
+ export const DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS = 1000;
31
+
32
+ export interface AdaptiveTimeoutInput {
33
+ /** The user-configured execution timeout (ms). The shrink never exceeds it. */
34
+ configuredMs: number;
35
+ /**
36
+ * p95 (or max) latency of this check's recent SUCCESSFUL runs, in ms.
37
+ * `undefined` means "no healthy baseline" => guardrail 1 (never shrink).
38
+ */
39
+ healthyBaselineMs: number | undefined;
40
+ /** Whether this check is classified as a consistent slot-hogging failure. */
41
+ isSuspect: boolean;
42
+ /** Whether this run is the periodic full-timeout recovery probe (guardrail 3). */
43
+ isRecoveryProbe: boolean;
44
+ /** Multiplier on the baseline. Defaults to {@link DEFAULT_TIMEOUT_SAFETY_FACTOR}. */
45
+ safetyFactor?: number;
46
+ /** Hard lower bound. Defaults to {@link DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS}. */
47
+ absoluteFloorMs?: number;
48
+ }
49
+
50
+ /**
51
+ * Resolve the effective execution timeout for a single (env-scoped) run.
52
+ * Returns `configuredMs` unchanged unless the check is a suspect slot-hogger
53
+ * WITH a healthy baseline AND this is not a recovery probe.
54
+ */
55
+ export function adaptiveTimeout(input: AdaptiveTimeoutInput): number {
56
+ const {
57
+ configuredMs,
58
+ healthyBaselineMs,
59
+ isSuspect,
60
+ isRecoveryProbe,
61
+ safetyFactor = DEFAULT_TIMEOUT_SAFETY_FACTOR,
62
+ absoluteFloorMs = DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
63
+ } = input;
64
+
65
+ // Guardrails 1 & 3, and the non-suspect fast path: use the full timeout.
66
+ if (!isSuspect || isRecoveryProbe || healthyBaselineMs === undefined) {
67
+ return configuredMs;
68
+ }
69
+
70
+ // Shrink toward the target's own healthy latency, clamped to
71
+ // [absoluteFloor, configured]. For a 10s-healthy check this stays >= ~15s;
72
+ // for a 200ms-healthy check it drops to the floor (~1s).
73
+ const shrunk = Math.round(healthyBaselineMs * safetyFactor);
74
+ return Math.min(configuredMs, Math.max(absoluteFloorMs, shrunk));
75
+ }
@@ -20,6 +20,8 @@ const unhealthyStatuses: HealthcheckSignalStatuses = {
20
20
  configurationName: "Ping",
21
21
  status: "unhealthy",
22
22
  runsConsidered: 5,
23
+ sliceCount: 1,
24
+ failingSliceCount: 1,
23
25
  },
24
26
  ],
25
27
  },
@@ -10,6 +10,7 @@ import {
10
10
  assignmentArtifactType,
11
11
  checkFailedTrigger,
12
12
  createHealthCheckActions,
13
+ type HealthCheckActionDeps,
13
14
  healthCheckTriggers,
14
15
  systemDegradedTrigger,
15
16
  systemHealthChangedTrigger,
@@ -146,13 +147,18 @@ describe("assignmentArtifactType", () => {
146
147
 
147
148
  function makeService(args: {
148
149
  setAssignmentEnabledReturn?: boolean;
150
+ enqueueEnvironmentIds?: (string | null)[];
149
151
  }): HealthCheckService & { setMock: ReturnType<typeof mock> } {
150
152
  const setMock = mock(
151
153
  async (_sysId: string, _cfgId: string, _enabled: boolean) =>
152
154
  args.setAssignmentEnabledReturn ?? true,
153
155
  );
156
+ const resolveEnqueueEnvironmentIds = mock(
157
+ async () => args.enqueueEnvironmentIds ?? [null],
158
+ );
154
159
  return {
155
160
  setAssignmentEnabled: setMock,
161
+ resolveEnqueueEnvironmentIds,
156
162
  setMock,
157
163
  } as unknown as HealthCheckService & { setMock: ReturnType<typeof mock> };
158
164
  }
@@ -174,6 +180,11 @@ function makeQueueManager(): QueueEnqueueRecorder {
174
180
  return { queueManager, enqueueMock };
175
181
  }
176
182
 
183
+ // The actions only need a catalog client shape for `run_now`, which delegates
184
+ // environment resolution to the service mock, so a bare stub suffices.
185
+ const catalogClientStub =
186
+ {} as unknown as HealthCheckActionDeps["catalogClient"];
187
+
177
188
  describe("healthcheck.run_now", () => {
178
189
  it("enqueues a one-off job and emits an enqueued=true artifact", async () => {
179
190
  const service = makeService({});
@@ -182,6 +193,7 @@ describe("healthcheck.run_now", () => {
182
193
  const [runNow] = createHealthCheckActions({
183
194
  service,
184
195
  queueManager,
196
+ catalogClient: catalogClientStub,
185
197
  emitHook: emitHook as never,
186
198
  });
187
199
 
@@ -198,10 +210,42 @@ describe("healthcheck.run_now", () => {
198
210
  expect(enqueueMock.mock.calls[0]![0]).toEqual({
199
211
  configId: "cfg-1",
200
212
  systemId: "sys-1",
213
+ environmentId: null,
201
214
  });
202
215
  // run_now doesn't mutate any DB row → no hook to emit.
203
216
  expect(emitHook).not.toHaveBeenCalled();
204
217
  });
218
+
219
+ it("enqueues one job per effective environment slice", async () => {
220
+ const service = makeService({ enqueueEnvironmentIds: ["prod", "staging"] });
221
+ const { queueManager, enqueueMock } = makeQueueManager();
222
+ const emitHook = mock(async (_hook: unknown, _payload: unknown) => {});
223
+ const [runNow] = createHealthCheckActions({
224
+ service,
225
+ queueManager,
226
+ catalogClient: catalogClientStub,
227
+ emitHook: emitHook as never,
228
+ });
229
+
230
+ const result = await runNow!.execute({
231
+ ...ctxBase,
232
+ consumedArtifacts: {},
233
+ config: { systemId: "sys-1", configurationId: "cfg-1" } as never,
234
+ });
235
+
236
+ expect(result.success).toBe(true);
237
+ expect(enqueueMock).toHaveBeenCalledTimes(2);
238
+ expect(enqueueMock.mock.calls[0]![0]).toEqual({
239
+ configId: "cfg-1",
240
+ systemId: "sys-1",
241
+ environmentId: "prod",
242
+ });
243
+ expect(enqueueMock.mock.calls[1]![0]).toEqual({
244
+ configId: "cfg-1",
245
+ systemId: "sys-1",
246
+ environmentId: "staging",
247
+ });
248
+ });
205
249
  });
206
250
 
207
251
  describe("healthcheck.enable_assignment", () => {
@@ -212,6 +256,7 @@ describe("healthcheck.enable_assignment", () => {
212
256
  const [, enable] = createHealthCheckActions({
213
257
  service,
214
258
  queueManager,
259
+ catalogClient: catalogClientStub,
215
260
  emitHook: emitHook as never,
216
261
  });
217
262
 
@@ -236,6 +281,7 @@ describe("healthcheck.enable_assignment", () => {
236
281
  const [, enable] = createHealthCheckActions({
237
282
  service,
238
283
  queueManager,
284
+ catalogClient: catalogClientStub,
239
285
  emitHook: emitHook as never,
240
286
  });
241
287
 
@@ -260,6 +306,7 @@ describe("healthcheck.disable_assignment", () => {
260
306
  const [, , disable] = createHealthCheckActions({
261
307
  service,
262
308
  queueManager,
309
+ catalogClient: catalogClientStub,
263
310
  emitHook: emitHook as never,
264
311
  });
265
312
 
@@ -26,6 +26,8 @@
26
26
  import { z } from "zod";
27
27
  import { Versioned, type Hook } from "@checkstack/backend-api";
28
28
  import type { QueueManager } from "@checkstack/queue-api";
29
+ import type { InferClient } from "@checkstack/common";
30
+ import type { CatalogApi } from "@checkstack/catalog-common";
29
31
  import type {
30
32
  ActionDefinition,
31
33
  TriggerDefinition,
@@ -234,6 +236,7 @@ export const assignmentArtifactType = {
234
236
  export interface HealthCheckActionDeps {
235
237
  service: HealthCheckService;
236
238
  queueManager: QueueManager;
239
+ catalogClient: InferClient<typeof CatalogApi>;
237
240
  emitHook: <T>(hook: Hook<T>, payload: T) => Promise<void>;
238
241
  }
239
242
 
@@ -256,12 +259,25 @@ export function createHealthCheckActions(
256
259
  const queue = deps.queueManager.getQueue<HealthCheckJobPayload>(
257
260
  HEALTH_CHECK_QUEUE,
258
261
  );
259
- await queue.enqueue({
260
- configId: config.configurationId,
262
+ // Enqueue one one-off job per effective environment slice so a manual
263
+ // run covers exactly the same slices the recurring schedule does. An
264
+ // assignment with no effective environments enqueues a single env-less
265
+ // run (`environmentId: null`).
266
+ const environmentIds = await deps.service.resolveEnqueueEnvironmentIds({
261
267
  systemId: config.systemId,
268
+ configurationId: config.configurationId,
269
+ catalogClient: deps.catalogClient,
270
+ logger,
262
271
  });
272
+ for (const environmentId of environmentIds) {
273
+ await queue.enqueue({
274
+ configId: config.configurationId,
275
+ systemId: config.systemId,
276
+ environmentId,
277
+ });
278
+ }
263
279
  logger.info(
264
- `Automation enqueued run for ${config.systemId}:${config.configurationId}`,
280
+ `Automation enqueued ${environmentIds.length} run(s) for ${config.systemId}:${config.configurationId}`,
265
281
  );
266
282
  return {
267
283
  success: true,
@@ -0,0 +1,89 @@
1
+ import { describe, it, expect } from "bun:test";
2
+ import { buildHealthTransitionNotification } from "./health-notification-content";
3
+
4
+ describe("buildHealthTransitionNotification", () => {
5
+ const base = {
6
+ systemId: "sys-1",
7
+ systemName: "Payments API",
8
+ configurationId: "cfg-9",
9
+ checkName: "HTTP 200 probe",
10
+ newStatus: "unhealthy" as const,
11
+ };
12
+
13
+ it("names the failing check in the body for an unhealthy transition", () => {
14
+ const payload = buildHealthTransitionNotification({
15
+ ...base,
16
+ transition: "escalation",
17
+ });
18
+ expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
19
+ expect(payload.body).toContain("**Payments API**");
20
+ expect(payload.importance).toBe("critical");
21
+ });
22
+
23
+ it("names the failing check for a degraded transition", () => {
24
+ const payload = buildHealthTransitionNotification({
25
+ ...base,
26
+ newStatus: "degraded",
27
+ transition: "escalation",
28
+ });
29
+ expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
30
+ expect(payload.importance).toBe("warning");
31
+ });
32
+
33
+ it("pushes a healthcheck.healthcheck subject alongside the system subject", () => {
34
+ const payload = buildHealthTransitionNotification({
35
+ ...base,
36
+ transition: "escalation",
37
+ });
38
+ const subjects = payload.subjects ?? [];
39
+ expect(subjects).toHaveLength(2);
40
+ expect(subjects[0]).toMatchObject({
41
+ kind: "catalog.system",
42
+ id: "sys-1",
43
+ name: "Payments API",
44
+ });
45
+ expect(subjects[1]).toMatchObject({
46
+ kind: "healthcheck.healthcheck",
47
+ id: "cfg-9",
48
+ name: "HTTP 200 probe",
49
+ status: "unhealthy",
50
+ });
51
+ // Check subject deep-links to its run history.
52
+ expect(subjects[1]?.url).toContain("sys-1");
53
+ expect(subjects[1]?.url).toContain("cfg-9");
54
+ });
55
+
56
+ it("falls back to the configuration id when no name is resolved", () => {
57
+ const payload = buildHealthTransitionNotification({
58
+ ...base,
59
+ checkName: "cfg-9",
60
+ transition: "escalation",
61
+ });
62
+ expect(payload.body).toContain('Health check **"cfg-9"**');
63
+ expect((payload.subjects ?? [])[1]).toMatchObject({ name: "cfg-9" });
64
+ });
65
+
66
+ it("qualifies the body with the environment name when env-scoped", () => {
67
+ const payload = buildHealthTransitionNotification({
68
+ ...base,
69
+ transition: "escalation",
70
+ environmentId: "env-prod",
71
+ environmentName: "Production",
72
+ });
73
+ expect(payload.body).toContain("in environment **Production**");
74
+ expect(payload.title).toContain("(Production)");
75
+ });
76
+
77
+ it("stays system-level and omits the check subject on recovery", () => {
78
+ const payload = buildHealthTransitionNotification({
79
+ ...base,
80
+ newStatus: "healthy",
81
+ transition: "recovery",
82
+ });
83
+ expect(payload.body).not.toContain("Health check **");
84
+ expect(payload.importance).toBe("info");
85
+ const subjects = payload.subjects ?? [];
86
+ expect(subjects).toHaveLength(1);
87
+ expect(subjects[0]).toMatchObject({ kind: "catalog.system" });
88
+ });
89
+ });
@@ -0,0 +1,138 @@
1
+ import { resolveRoute, type InferClient } from "@checkstack/common";
2
+ import { catalogRoutes, createSystemSubject } from "@checkstack/catalog-common";
3
+ import type { NotificationApi } from "@checkstack/notification-common";
4
+ import {
5
+ createHealthcheckSubject,
6
+ healthcheckRoutes,
7
+ systemHealthCollapseKey,
8
+ healthcheckSystemSubscription,
9
+ type HealthCheckStatus,
10
+ } from "@checkstack/healthcheck-common";
11
+ import type { TransitionKind } from "./notification-policy";
12
+
13
+ /** The subset of `notifyForSubscription`'s input this builder produces. */
14
+ type NotifyForSubscriptionInput = Parameters<
15
+ InferClient<typeof NotificationApi>["notifyForSubscription"]
16
+ >[0];
17
+
18
+ /**
19
+ * Inputs to {@link buildHealthTransitionNotification}. Pure data only - the
20
+ * catalog client is unused here (parents are resolved server-side) and thus
21
+ * omitted; every field is derived before the call site in the queue executor.
22
+ */
23
+ export interface HealthTransitionNotificationInput {
24
+ transition: Exclude<TransitionKind, "none">;
25
+ systemId: string;
26
+ systemName: string;
27
+ configurationId: string;
28
+ /** Resolved display name of the check that drove the transition. */
29
+ checkName: string;
30
+ newStatus: HealthCheckStatus;
31
+ /** Concrete env id for a per-env slice, null/undefined for the system rollup. */
32
+ environmentId?: string | null;
33
+ /** Human-readable env name for the body/title. */
34
+ environmentName?: string;
35
+ }
36
+
37
+ /**
38
+ * Build the notification payload for a health-state transition. Pure and
39
+ * side-effect free so it can be unit-tested directly. Extracted from
40
+ * `notifyStateChange` so the body/title/subject wording (which now NAMES the
41
+ * failing check and pushes a `healthcheck.healthcheck` subject) is verifiable
42
+ * without booting the whole queue executor.
43
+ *
44
+ * Recovery bodies stay system-level (the whole system is green again; naming
45
+ * one check would mislead) and omit the check subject. Failing transitions
46
+ * (escalation / de-escalation) name the check in the body and add it as a
47
+ * subject deep-linked to its run history.
48
+ */
49
+ export function buildHealthTransitionNotification(
50
+ input: HealthTransitionNotificationInput,
51
+ ): NotifyForSubscriptionInput {
52
+ const {
53
+ transition,
54
+ systemId,
55
+ systemName,
56
+ configurationId,
57
+ checkName,
58
+ newStatus,
59
+ environmentId,
60
+ environmentName,
61
+ } = input;
62
+
63
+ const envScoped = typeof environmentId === "string";
64
+ const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
65
+ const envQualifier = envScoped
66
+ ? ` in environment **${environmentName ?? environmentId}**`
67
+ : "";
68
+
69
+ let title: string;
70
+ let body: string;
71
+ let importance: "info" | "warning" | "critical";
72
+
73
+ if (transition === "recovery") {
74
+ title = `System health restored${envSuffix}: ${systemName}`;
75
+ body = envScoped
76
+ ? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
77
+ : `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
78
+ importance = "info";
79
+ } else if (newStatus === "unhealthy") {
80
+ title = `System health critical${envSuffix}: ${systemName}`;
81
+ body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system is unhealthy and may be down${envScoped ? " in that environment" : ""}.`;
82
+ importance = "critical";
83
+ } else {
84
+ // degraded - either an escalation from healthy or a partial recovery
85
+ title = `System health degraded${envSuffix}: ${systemName}`;
86
+ body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system may be experiencing issues${envScoped ? " in that environment" : ""}.`;
87
+ importance = "warning";
88
+ }
89
+
90
+ const systemDetailPath = resolveRoute(catalogRoutes.routes.systemDetail, {
91
+ systemId,
92
+ });
93
+ // Recovery lands on the default (all) view; failing transitions deep-link
94
+ // operators into the failing-checks filter so they can debug immediately.
95
+ const actionUrl =
96
+ transition === "recovery"
97
+ ? systemDetailPath
98
+ : `${systemDetailPath}?filter=failing`;
99
+ const actionLabel =
100
+ transition === "recovery" ? "View System" : "View failing checks";
101
+
102
+ return {
103
+ specId: healthcheckSystemSubscription.specId,
104
+ resourceKeys: [systemId],
105
+ title,
106
+ body,
107
+ importance,
108
+ action: { label: actionLabel, url: actionUrl },
109
+ // Env-qualified collapse key so two failing envs of one system generate
110
+ // two independent notification cards (one per env) instead of merging.
111
+ collapseKey: envScoped
112
+ ? systemHealthCollapseKey(systemId, environmentId)
113
+ : systemHealthCollapseKey(systemId),
114
+ subjects: [
115
+ createSystemSubject({
116
+ id: systemId,
117
+ name: systemName,
118
+ url: systemDetailPath,
119
+ status: newStatus,
120
+ }),
121
+ // Name the failing check as its own subject for every non-recovery
122
+ // transition, deep-linked to its run history. Omitted on recovery.
123
+ ...(transition === "recovery"
124
+ ? []
125
+ : [
126
+ createHealthcheckSubject({
127
+ id: configurationId,
128
+ name: checkName,
129
+ url: resolveRoute(healthcheckRoutes.routes.historyDetail, {
130
+ systemId,
131
+ configurationId,
132
+ }),
133
+ status: newStatus,
134
+ }),
135
+ ]),
136
+ ],
137
+ };
138
+ }
@@ -20,6 +20,34 @@ import { Versioned } from "@checkstack/backend-api";
20
20
  * System → healthchecks extension in isolation with mock services.
21
21
  */
22
22
 
23
+ // The System extension's reconcile converges the per-env recurring jobs via
24
+ // `reconcileHealthCheckJobs`. That path has its own dedicated tests
25
+ // (schedule-reconciler.test.ts); here we feed it EMPTY db/catalog/queue stubs so
26
+ // it is a safe no-op and these tests can focus on the association logic.
27
+ // (A `mock.module` would leak process-wide and break schedule-reconciler.test.)
28
+ function emptyReconcileDbStub() {
29
+ return {
30
+ select: () => ({
31
+ from: () => ({
32
+ innerJoin: () => ({ where: () => Promise.resolve([]) }),
33
+ groupBy: () => Promise.resolve([]),
34
+ }),
35
+ }),
36
+ } as never;
37
+ }
38
+ function emptyReconcileQueueManager() {
39
+ return {
40
+ getQueue: () => ({
41
+ scheduleRecurring: async () => "job-123",
42
+ listRecurringJobs: async () => [],
43
+ getRecurringJobDetails: async () => undefined,
44
+ cancelRecurring: async () => {},
45
+ }),
46
+ } as never;
47
+ }
48
+ const emptyCatalogClient = () =>
49
+ ({ resolveSystemEnvironments: async () => [] }) as never;
50
+
23
51
  // ─── Mock Healthcheck Service ──────────────────────────────────────────────
24
52
 
25
53
  interface MockConfig {
@@ -231,7 +259,9 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
231
259
  createService: () => mockService as any,
232
260
  getHealthCheckRegistry: () => mockHCRegistry as any,
233
261
  getCollectorRegistry: () => mockCollectorRegistry as any,
234
- getQueueManager: () => ({ getQueue: () => ({ scheduleRecurring: async () => "job-123" }) } as any),
262
+ getQueueManager: () => emptyReconcileQueueManager(),
263
+ getDb: () => emptyReconcileDbStub(),
264
+ getCatalogClient: () => emptyCatalogClient(),
235
265
  };
236
266
  return buildHealthcheckKind(mockDeps);
237
267
  }
@@ -594,7 +624,9 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
594
624
  }) as never,
595
625
  getHealthCheckRegistry: () => createMockHealthCheckRegistry() as never,
596
626
  getCollectorRegistry: () => createMockCollectorRegistry() as never,
597
- getQueueManager: () => ({ getQueue: () => ({ scheduleRecurring: async () => "job-123" }) } as any),
627
+ getQueueManager: () => emptyReconcileQueueManager(),
628
+ getDb: () => emptyReconcileDbStub(),
629
+ getCatalogClient: () => emptyCatalogClient(),
598
630
  });
599
631
  }
600
632
 
@@ -25,7 +25,11 @@ import {
25
25
  enumField,
26
26
  } from "@checkstack/backend-api";
27
27
  import type { QueueManager } from "@checkstack/queue-api";
28
- import { scheduleHealthCheck } from "./queue-executor";
28
+ import type { SafeDatabase } from "@checkstack/backend-api";
29
+ import type { InferClient } from "@checkstack/common";
30
+ import type { CatalogApi } from "@checkstack/catalog-common";
31
+ import type * as schema from "./schema";
32
+ import { reconcileHealthCheckJobs } from "./schedule-reconciler";
29
33
 
30
34
  /**
31
35
  * Lazy accessor functions — populated during init(), consumed during reconcile.
@@ -37,6 +41,8 @@ interface HealthcheckGitOpsKindsDeps {
37
41
  getHealthCheckRegistry: () => HealthCheckRegistry;
38
42
  getCollectorRegistry: () => CollectorRegistry;
39
43
  getQueueManager: () => QueueManager;
44
+ getDb: () => SafeDatabase<typeof schema>;
45
+ getCatalogClient: () => InferClient<typeof CatalogApi>;
40
46
  }
41
47
 
42
48
  // ─── Healthcheck Spec Schema ───────────────────────────────────────────────
@@ -382,18 +388,16 @@ export function buildSystemHealthcheckExtension(
382
388
  notificationPolicy,
383
389
  });
384
390
 
385
- // Retrieve config to get the interval for scheduling
386
- const config = await service.getConfiguration(configId);
387
- if (config) {
388
- await scheduleHealthCheck({
389
- queueManager: deps.getQueueManager(),
390
- payload: {
391
- configId,
392
- systemId: systemEntityId,
393
- },
394
- intervalSeconds: config.intervalSeconds,
395
- });
396
- }
391
+ // Reconcile this system's per-env recurring jobs so the new
392
+ // association starts probing right away (system-scoped: add/update
393
+ // only, orphan cleanup is owned by the periodic full reconcile).
394
+ await reconcileHealthCheckJobs({
395
+ db: deps.getDb(),
396
+ queueManager: deps.getQueueManager(),
397
+ catalogClient: deps.getCatalogClient(),
398
+ logger: context.logger,
399
+ systemId: systemEntityId,
400
+ });
397
401
 
398
402
  context.logger.info(
399
403
  `GitOps: associated ${entry.ref.kind} "${entry.ref.name}" (${configId}) with System "${entity.metadata.name}" and scheduled execution`,