@checkstack/healthcheck-backend 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +559 -0
- package/package.json +32 -29
- package/src/adaptive-timeout.test.ts +91 -0
- package/src/adaptive-timeout.ts +75 -0
- package/src/ai/system-signals-contributor.test.ts +2 -0
- package/src/automations.test.ts +47 -0
- package/src/automations.ts +19 -3
- package/src/health-notification-content.test.ts +89 -0
- package/src/health-notification-content.ts +138 -0
- package/src/healthcheck-gitops-kinds.test.ts +34 -2
- package/src/healthcheck-gitops-kinds.ts +17 -13
- package/src/index.ts +58 -6
- package/src/migration-chain-contract.test.ts +7 -1
- package/src/notification-policy.test.ts +19 -0
- package/src/notification-policy.ts +26 -0
- package/src/queue-executor.test.ts +391 -338
- package/src/queue-executor.ts +426 -362
- package/src/realtime-aggregation.ts +9 -2
- package/src/rollup-consumer.test.ts +191 -0
- package/src/rollup-consumer.ts +160 -0
- package/src/router.ts +46 -13
- package/src/schedule-jitter.test.ts +69 -0
- package/src/schedule-jitter.ts +50 -0
- package/src/schedule-reconciler.it.test.ts +453 -0
- package/src/schedule-reconciler.test.ts +418 -0
- package/src/schedule-reconciler.ts +304 -0
- package/src/service-batching.test.ts +106 -0
- package/src/service-bulk-counts.it.test.ts +144 -0
- package/src/service-bulk-run-stats.it.test.ts +197 -0
- package/src/service-ordering.test.ts +10 -2
- package/src/service-paused-filter.test.ts +27 -7
- package/src/service-rollup-worst-wins.test.ts +221 -124
- package/src/service.ts +557 -266
- package/src/slow-check-admission.test.ts +184 -0
- package/src/slow-check-admission.ts +101 -0
- package/src/slow-check-classifier.test.ts +155 -0
- package/src/slow-check-classifier.ts +137 -0
- package/src/slow-check-config.ts +102 -0
- package/src/status-page/rollup.test.ts +40 -0
- package/src/status-page/rollup.ts +27 -0
- package/src/status-page/widgets.test.ts +303 -0
- package/src/status-page/widgets.ts +155 -39
- package/src/suspect-lane.test.ts +50 -0
- package/src/suspect-lane.ts +61 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
adaptiveTimeout,
|
|
4
|
+
DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
|
|
5
|
+
} from "./adaptive-timeout";
|
|
6
|
+
|
|
7
|
+
describe("adaptiveTimeout", () => {
|
|
8
|
+
const configuredMs = 30_000;
|
|
9
|
+
|
|
10
|
+
test("guardrail 1: no baseline => never shrink", () => {
|
|
11
|
+
expect(
|
|
12
|
+
adaptiveTimeout({
|
|
13
|
+
configuredMs,
|
|
14
|
+
healthyBaselineMs: undefined,
|
|
15
|
+
isSuspect: true,
|
|
16
|
+
isRecoveryProbe: false,
|
|
17
|
+
}),
|
|
18
|
+
).toBe(configuredMs);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("non-suspect check keeps the full configured timeout", () => {
|
|
22
|
+
expect(
|
|
23
|
+
adaptiveTimeout({
|
|
24
|
+
configuredMs,
|
|
25
|
+
healthyBaselineMs: 200,
|
|
26
|
+
isSuspect: false,
|
|
27
|
+
isRecoveryProbe: false,
|
|
28
|
+
}),
|
|
29
|
+
).toBe(configuredMs);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("guardrail 3: recovery probe always uses the full configured timeout", () => {
|
|
33
|
+
expect(
|
|
34
|
+
adaptiveTimeout({
|
|
35
|
+
configuredMs,
|
|
36
|
+
healthyBaselineMs: 200,
|
|
37
|
+
isSuspect: true,
|
|
38
|
+
isRecoveryProbe: true,
|
|
39
|
+
}),
|
|
40
|
+
).toBe(configuredMs);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("fast healthy check shrinks toward the floor (200ms => ~1s)", () => {
|
|
44
|
+
// 200 * 1.5 = 300 < floor => clamped to the absolute floor.
|
|
45
|
+
expect(
|
|
46
|
+
adaptiveTimeout({
|
|
47
|
+
configuredMs,
|
|
48
|
+
healthyBaselineMs: 200,
|
|
49
|
+
isSuspect: true,
|
|
50
|
+
isRecoveryProbe: false,
|
|
51
|
+
}),
|
|
52
|
+
).toBe(DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("deadlock guard: a slow-but-healthy check (10s) never shrinks below its own latency", () => {
|
|
56
|
+
// 10_000 * 1.5 = 15_000: a recovering 10s run still passes at 15s.
|
|
57
|
+
const t = adaptiveTimeout({
|
|
58
|
+
configuredMs,
|
|
59
|
+
healthyBaselineMs: 10_000,
|
|
60
|
+
isSuspect: true,
|
|
61
|
+
isRecoveryProbe: false,
|
|
62
|
+
});
|
|
63
|
+
expect(t).toBe(15_000);
|
|
64
|
+
expect(t).toBeGreaterThan(10_000);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("shrink never exceeds the configured timeout", () => {
|
|
68
|
+
// baseline*factor would exceed configured => clamp down to configured.
|
|
69
|
+
expect(
|
|
70
|
+
adaptiveTimeout({
|
|
71
|
+
configuredMs: 5_000,
|
|
72
|
+
healthyBaselineMs: 8_000,
|
|
73
|
+
isSuspect: true,
|
|
74
|
+
isRecoveryProbe: false,
|
|
75
|
+
}),
|
|
76
|
+
).toBe(5_000);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("honors custom safetyFactor and absoluteFloor", () => {
|
|
80
|
+
expect(
|
|
81
|
+
adaptiveTimeout({
|
|
82
|
+
configuredMs,
|
|
83
|
+
healthyBaselineMs: 1_000,
|
|
84
|
+
isSuspect: true,
|
|
85
|
+
isRecoveryProbe: false,
|
|
86
|
+
safetyFactor: 2,
|
|
87
|
+
absoluteFloorMs: 500,
|
|
88
|
+
}),
|
|
89
|
+
).toBe(2_000);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adaptive execution timeout for slot-hogging health checks.
|
|
3
|
+
*
|
|
4
|
+
* A check that hangs holds its concurrency slot for the FULL configured timeout
|
|
5
|
+
* (see `queue-executor.ts` — the per-env `Promise.race` frees the slot only when
|
|
6
|
+
* the timeout fires). When a target is consistently timing out, we can free that
|
|
7
|
+
* slot sooner by probing it with a SHORTER timeout — but only down to a value at
|
|
8
|
+
* which a genuinely HEALTHY run of this same check would still succeed, or we
|
|
9
|
+
* would abort even a recovering target and never let it go healthy again.
|
|
10
|
+
*
|
|
11
|
+
* The floor is therefore derived from the check's OWN measured healthy latency,
|
|
12
|
+
* never a global constant. Four guardrails make a recovery deadlock impossible:
|
|
13
|
+
*
|
|
14
|
+
* 1. No baseline (no recent healthy run) => NO shrink. Without evidence of the
|
|
15
|
+
* target's healthy latency we cannot tell "hung" from "legitimately slow and
|
|
16
|
+
* about to succeed", so we keep the full configured timeout.
|
|
17
|
+
* 2. The baseline is computed from SUCCESSFUL runs only (caller's job), so a
|
|
18
|
+
* timed-out run (latency ~= timeout) can never ratchet the floor downward.
|
|
19
|
+
* 3. The periodic recovery probe passes `isRecoveryProbe: true` and always gets
|
|
20
|
+
* the FULL configured timeout — so a genuinely-slow-but-healthy target (e.g.
|
|
21
|
+
* a 10s computation) is always eventually re-measured at its real latency.
|
|
22
|
+
* This is the by-construction deadlock breaker.
|
|
23
|
+
* 4. A single healthy run clears the suspect classification upstream, so the
|
|
24
|
+
* full timeout is restored immediately (hysteresis).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** Default multiplier applied to the healthy-latency baseline. */
|
|
28
|
+
export const DEFAULT_TIMEOUT_SAFETY_FACTOR = 1.5;
|
|
29
|
+
/** Default absolute lower bound; the timeout is never shrunk below this. */
|
|
30
|
+
export const DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS = 1000;
|
|
31
|
+
|
|
32
|
+
export interface AdaptiveTimeoutInput {
|
|
33
|
+
/** The user-configured execution timeout (ms). The shrink never exceeds it. */
|
|
34
|
+
configuredMs: number;
|
|
35
|
+
/**
|
|
36
|
+
* p95 (or max) latency of this check's recent SUCCESSFUL runs, in ms.
|
|
37
|
+
* `undefined` means "no healthy baseline" => guardrail 1 (never shrink).
|
|
38
|
+
*/
|
|
39
|
+
healthyBaselineMs: number | undefined;
|
|
40
|
+
/** Whether this check is classified as a consistent slot-hogging failure. */
|
|
41
|
+
isSuspect: boolean;
|
|
42
|
+
/** Whether this run is the periodic full-timeout recovery probe (guardrail 3). */
|
|
43
|
+
isRecoveryProbe: boolean;
|
|
44
|
+
/** Multiplier on the baseline. Defaults to {@link DEFAULT_TIMEOUT_SAFETY_FACTOR}. */
|
|
45
|
+
safetyFactor?: number;
|
|
46
|
+
/** Hard lower bound. Defaults to {@link DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS}. */
|
|
47
|
+
absoluteFloorMs?: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Resolve the effective execution timeout for a single (env-scoped) run.
|
|
52
|
+
* Returns `configuredMs` unchanged unless the check is a suspect slot-hogger
|
|
53
|
+
* WITH a healthy baseline AND this is not a recovery probe.
|
|
54
|
+
*/
|
|
55
|
+
export function adaptiveTimeout(input: AdaptiveTimeoutInput): number {
|
|
56
|
+
const {
|
|
57
|
+
configuredMs,
|
|
58
|
+
healthyBaselineMs,
|
|
59
|
+
isSuspect,
|
|
60
|
+
isRecoveryProbe,
|
|
61
|
+
safetyFactor = DEFAULT_TIMEOUT_SAFETY_FACTOR,
|
|
62
|
+
absoluteFloorMs = DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
|
|
63
|
+
} = input;
|
|
64
|
+
|
|
65
|
+
// Guardrails 1 & 3, and the non-suspect fast path: use the full timeout.
|
|
66
|
+
if (!isSuspect || isRecoveryProbe || healthyBaselineMs === undefined) {
|
|
67
|
+
return configuredMs;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Shrink toward the target's own healthy latency, clamped to
|
|
71
|
+
// [absoluteFloor, configured]. For a 10s-healthy check this stays >= ~15s;
|
|
72
|
+
// for a 200ms-healthy check it drops to the floor (~1s).
|
|
73
|
+
const shrunk = Math.round(healthyBaselineMs * safetyFactor);
|
|
74
|
+
return Math.min(configuredMs, Math.max(absoluteFloorMs, shrunk));
|
|
75
|
+
}
|
package/src/automations.test.ts
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
assignmentArtifactType,
|
|
11
11
|
checkFailedTrigger,
|
|
12
12
|
createHealthCheckActions,
|
|
13
|
+
type HealthCheckActionDeps,
|
|
13
14
|
healthCheckTriggers,
|
|
14
15
|
systemDegradedTrigger,
|
|
15
16
|
systemHealthChangedTrigger,
|
|
@@ -146,13 +147,18 @@ describe("assignmentArtifactType", () => {
|
|
|
146
147
|
|
|
147
148
|
function makeService(args: {
|
|
148
149
|
setAssignmentEnabledReturn?: boolean;
|
|
150
|
+
enqueueEnvironmentIds?: (string | null)[];
|
|
149
151
|
}): HealthCheckService & { setMock: ReturnType<typeof mock> } {
|
|
150
152
|
const setMock = mock(
|
|
151
153
|
async (_sysId: string, _cfgId: string, _enabled: boolean) =>
|
|
152
154
|
args.setAssignmentEnabledReturn ?? true,
|
|
153
155
|
);
|
|
156
|
+
const resolveEnqueueEnvironmentIds = mock(
|
|
157
|
+
async () => args.enqueueEnvironmentIds ?? [null],
|
|
158
|
+
);
|
|
154
159
|
return {
|
|
155
160
|
setAssignmentEnabled: setMock,
|
|
161
|
+
resolveEnqueueEnvironmentIds,
|
|
156
162
|
setMock,
|
|
157
163
|
} as unknown as HealthCheckService & { setMock: ReturnType<typeof mock> };
|
|
158
164
|
}
|
|
@@ -174,6 +180,11 @@ function makeQueueManager(): QueueEnqueueRecorder {
|
|
|
174
180
|
return { queueManager, enqueueMock };
|
|
175
181
|
}
|
|
176
182
|
|
|
183
|
+
// The actions only need a catalog client shape for `run_now`, which delegates
|
|
184
|
+
// environment resolution to the service mock, so a bare stub suffices.
|
|
185
|
+
const catalogClientStub =
|
|
186
|
+
{} as unknown as HealthCheckActionDeps["catalogClient"];
|
|
187
|
+
|
|
177
188
|
describe("healthcheck.run_now", () => {
|
|
178
189
|
it("enqueues a one-off job and emits an enqueued=true artifact", async () => {
|
|
179
190
|
const service = makeService({});
|
|
@@ -182,6 +193,7 @@ describe("healthcheck.run_now", () => {
|
|
|
182
193
|
const [runNow] = createHealthCheckActions({
|
|
183
194
|
service,
|
|
184
195
|
queueManager,
|
|
196
|
+
catalogClient: catalogClientStub,
|
|
185
197
|
emitHook: emitHook as never,
|
|
186
198
|
});
|
|
187
199
|
|
|
@@ -198,10 +210,42 @@ describe("healthcheck.run_now", () => {
|
|
|
198
210
|
expect(enqueueMock.mock.calls[0]![0]).toEqual({
|
|
199
211
|
configId: "cfg-1",
|
|
200
212
|
systemId: "sys-1",
|
|
213
|
+
environmentId: null,
|
|
201
214
|
});
|
|
202
215
|
// run_now doesn't mutate any DB row → no hook to emit.
|
|
203
216
|
expect(emitHook).not.toHaveBeenCalled();
|
|
204
217
|
});
|
|
218
|
+
|
|
219
|
+
it("enqueues one job per effective environment slice", async () => {
|
|
220
|
+
const service = makeService({ enqueueEnvironmentIds: ["prod", "staging"] });
|
|
221
|
+
const { queueManager, enqueueMock } = makeQueueManager();
|
|
222
|
+
const emitHook = mock(async (_hook: unknown, _payload: unknown) => {});
|
|
223
|
+
const [runNow] = createHealthCheckActions({
|
|
224
|
+
service,
|
|
225
|
+
queueManager,
|
|
226
|
+
catalogClient: catalogClientStub,
|
|
227
|
+
emitHook: emitHook as never,
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const result = await runNow!.execute({
|
|
231
|
+
...ctxBase,
|
|
232
|
+
consumedArtifacts: {},
|
|
233
|
+
config: { systemId: "sys-1", configurationId: "cfg-1" } as never,
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
expect(result.success).toBe(true);
|
|
237
|
+
expect(enqueueMock).toHaveBeenCalledTimes(2);
|
|
238
|
+
expect(enqueueMock.mock.calls[0]![0]).toEqual({
|
|
239
|
+
configId: "cfg-1",
|
|
240
|
+
systemId: "sys-1",
|
|
241
|
+
environmentId: "prod",
|
|
242
|
+
});
|
|
243
|
+
expect(enqueueMock.mock.calls[1]![0]).toEqual({
|
|
244
|
+
configId: "cfg-1",
|
|
245
|
+
systemId: "sys-1",
|
|
246
|
+
environmentId: "staging",
|
|
247
|
+
});
|
|
248
|
+
});
|
|
205
249
|
});
|
|
206
250
|
|
|
207
251
|
describe("healthcheck.enable_assignment", () => {
|
|
@@ -212,6 +256,7 @@ describe("healthcheck.enable_assignment", () => {
|
|
|
212
256
|
const [, enable] = createHealthCheckActions({
|
|
213
257
|
service,
|
|
214
258
|
queueManager,
|
|
259
|
+
catalogClient: catalogClientStub,
|
|
215
260
|
emitHook: emitHook as never,
|
|
216
261
|
});
|
|
217
262
|
|
|
@@ -236,6 +281,7 @@ describe("healthcheck.enable_assignment", () => {
|
|
|
236
281
|
const [, enable] = createHealthCheckActions({
|
|
237
282
|
service,
|
|
238
283
|
queueManager,
|
|
284
|
+
catalogClient: catalogClientStub,
|
|
239
285
|
emitHook: emitHook as never,
|
|
240
286
|
});
|
|
241
287
|
|
|
@@ -260,6 +306,7 @@ describe("healthcheck.disable_assignment", () => {
|
|
|
260
306
|
const [, , disable] = createHealthCheckActions({
|
|
261
307
|
service,
|
|
262
308
|
queueManager,
|
|
309
|
+
catalogClient: catalogClientStub,
|
|
263
310
|
emitHook: emitHook as never,
|
|
264
311
|
});
|
|
265
312
|
|
package/src/automations.ts
CHANGED
|
@@ -26,6 +26,8 @@
|
|
|
26
26
|
import { z } from "zod";
|
|
27
27
|
import { Versioned, type Hook } from "@checkstack/backend-api";
|
|
28
28
|
import type { QueueManager } from "@checkstack/queue-api";
|
|
29
|
+
import type { InferClient } from "@checkstack/common";
|
|
30
|
+
import type { CatalogApi } from "@checkstack/catalog-common";
|
|
29
31
|
import type {
|
|
30
32
|
ActionDefinition,
|
|
31
33
|
TriggerDefinition,
|
|
@@ -234,6 +236,7 @@ export const assignmentArtifactType = {
|
|
|
234
236
|
export interface HealthCheckActionDeps {
|
|
235
237
|
service: HealthCheckService;
|
|
236
238
|
queueManager: QueueManager;
|
|
239
|
+
catalogClient: InferClient<typeof CatalogApi>;
|
|
237
240
|
emitHook: <T>(hook: Hook<T>, payload: T) => Promise<void>;
|
|
238
241
|
}
|
|
239
242
|
|
|
@@ -256,12 +259,25 @@ export function createHealthCheckActions(
|
|
|
256
259
|
const queue = deps.queueManager.getQueue<HealthCheckJobPayload>(
|
|
257
260
|
HEALTH_CHECK_QUEUE,
|
|
258
261
|
);
|
|
259
|
-
|
|
260
|
-
|
|
262
|
+
// Enqueue one one-off job per effective environment slice so a manual
|
|
263
|
+
// run covers exactly the same slices the recurring schedule does. An
|
|
264
|
+
// assignment with no effective environments enqueues a single env-less
|
|
265
|
+
// run (`environmentId: null`).
|
|
266
|
+
const environmentIds = await deps.service.resolveEnqueueEnvironmentIds({
|
|
261
267
|
systemId: config.systemId,
|
|
268
|
+
configurationId: config.configurationId,
|
|
269
|
+
catalogClient: deps.catalogClient,
|
|
270
|
+
logger,
|
|
262
271
|
});
|
|
272
|
+
for (const environmentId of environmentIds) {
|
|
273
|
+
await queue.enqueue({
|
|
274
|
+
configId: config.configurationId,
|
|
275
|
+
systemId: config.systemId,
|
|
276
|
+
environmentId,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
263
279
|
logger.info(
|
|
264
|
-
`Automation enqueued run for ${config.systemId}:${config.configurationId}`,
|
|
280
|
+
`Automation enqueued ${environmentIds.length} run(s) for ${config.systemId}:${config.configurationId}`,
|
|
265
281
|
);
|
|
266
282
|
return {
|
|
267
283
|
success: true,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { describe, it, expect } from "bun:test";
|
|
2
|
+
import { buildHealthTransitionNotification } from "./health-notification-content";
|
|
3
|
+
|
|
4
|
+
describe("buildHealthTransitionNotification", () => {
|
|
5
|
+
const base = {
|
|
6
|
+
systemId: "sys-1",
|
|
7
|
+
systemName: "Payments API",
|
|
8
|
+
configurationId: "cfg-9",
|
|
9
|
+
checkName: "HTTP 200 probe",
|
|
10
|
+
newStatus: "unhealthy" as const,
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
it("names the failing check in the body for an unhealthy transition", () => {
|
|
14
|
+
const payload = buildHealthTransitionNotification({
|
|
15
|
+
...base,
|
|
16
|
+
transition: "escalation",
|
|
17
|
+
});
|
|
18
|
+
expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
|
|
19
|
+
expect(payload.body).toContain("**Payments API**");
|
|
20
|
+
expect(payload.importance).toBe("critical");
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it("names the failing check for a degraded transition", () => {
|
|
24
|
+
const payload = buildHealthTransitionNotification({
|
|
25
|
+
...base,
|
|
26
|
+
newStatus: "degraded",
|
|
27
|
+
transition: "escalation",
|
|
28
|
+
});
|
|
29
|
+
expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
|
|
30
|
+
expect(payload.importance).toBe("warning");
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("pushes a healthcheck.healthcheck subject alongside the system subject", () => {
|
|
34
|
+
const payload = buildHealthTransitionNotification({
|
|
35
|
+
...base,
|
|
36
|
+
transition: "escalation",
|
|
37
|
+
});
|
|
38
|
+
const subjects = payload.subjects ?? [];
|
|
39
|
+
expect(subjects).toHaveLength(2);
|
|
40
|
+
expect(subjects[0]).toMatchObject({
|
|
41
|
+
kind: "catalog.system",
|
|
42
|
+
id: "sys-1",
|
|
43
|
+
name: "Payments API",
|
|
44
|
+
});
|
|
45
|
+
expect(subjects[1]).toMatchObject({
|
|
46
|
+
kind: "healthcheck.healthcheck",
|
|
47
|
+
id: "cfg-9",
|
|
48
|
+
name: "HTTP 200 probe",
|
|
49
|
+
status: "unhealthy",
|
|
50
|
+
});
|
|
51
|
+
// Check subject deep-links to its run history.
|
|
52
|
+
expect(subjects[1]?.url).toContain("sys-1");
|
|
53
|
+
expect(subjects[1]?.url).toContain("cfg-9");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("falls back to the configuration id when no name is resolved", () => {
|
|
57
|
+
const payload = buildHealthTransitionNotification({
|
|
58
|
+
...base,
|
|
59
|
+
checkName: "cfg-9",
|
|
60
|
+
transition: "escalation",
|
|
61
|
+
});
|
|
62
|
+
expect(payload.body).toContain('Health check **"cfg-9"**');
|
|
63
|
+
expect((payload.subjects ?? [])[1]).toMatchObject({ name: "cfg-9" });
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("qualifies the body with the environment name when env-scoped", () => {
|
|
67
|
+
const payload = buildHealthTransitionNotification({
|
|
68
|
+
...base,
|
|
69
|
+
transition: "escalation",
|
|
70
|
+
environmentId: "env-prod",
|
|
71
|
+
environmentName: "Production",
|
|
72
|
+
});
|
|
73
|
+
expect(payload.body).toContain("in environment **Production**");
|
|
74
|
+
expect(payload.title).toContain("(Production)");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("stays system-level and omits the check subject on recovery", () => {
|
|
78
|
+
const payload = buildHealthTransitionNotification({
|
|
79
|
+
...base,
|
|
80
|
+
newStatus: "healthy",
|
|
81
|
+
transition: "recovery",
|
|
82
|
+
});
|
|
83
|
+
expect(payload.body).not.toContain("Health check **");
|
|
84
|
+
expect(payload.importance).toBe("info");
|
|
85
|
+
const subjects = payload.subjects ?? [];
|
|
86
|
+
expect(subjects).toHaveLength(1);
|
|
87
|
+
expect(subjects[0]).toMatchObject({ kind: "catalog.system" });
|
|
88
|
+
});
|
|
89
|
+
});
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { resolveRoute, type InferClient } from "@checkstack/common";
|
|
2
|
+
import { catalogRoutes, createSystemSubject } from "@checkstack/catalog-common";
|
|
3
|
+
import type { NotificationApi } from "@checkstack/notification-common";
|
|
4
|
+
import {
|
|
5
|
+
createHealthcheckSubject,
|
|
6
|
+
healthcheckRoutes,
|
|
7
|
+
systemHealthCollapseKey,
|
|
8
|
+
healthcheckSystemSubscription,
|
|
9
|
+
type HealthCheckStatus,
|
|
10
|
+
} from "@checkstack/healthcheck-common";
|
|
11
|
+
import type { TransitionKind } from "./notification-policy";
|
|
12
|
+
|
|
13
|
+
/** The subset of `notifyForSubscription`'s input this builder produces. */
|
|
14
|
+
type NotifyForSubscriptionInput = Parameters<
|
|
15
|
+
InferClient<typeof NotificationApi>["notifyForSubscription"]
|
|
16
|
+
>[0];
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Inputs to {@link buildHealthTransitionNotification}. Pure data only - the
|
|
20
|
+
* catalog client is unused here (parents are resolved server-side) and thus
|
|
21
|
+
* omitted; every field is derived before the call site in the queue executor.
|
|
22
|
+
*/
|
|
23
|
+
export interface HealthTransitionNotificationInput {
|
|
24
|
+
transition: Exclude<TransitionKind, "none">;
|
|
25
|
+
systemId: string;
|
|
26
|
+
systemName: string;
|
|
27
|
+
configurationId: string;
|
|
28
|
+
/** Resolved display name of the check that drove the transition. */
|
|
29
|
+
checkName: string;
|
|
30
|
+
newStatus: HealthCheckStatus;
|
|
31
|
+
/** Concrete env id for a per-env slice, null/undefined for the system rollup. */
|
|
32
|
+
environmentId?: string | null;
|
|
33
|
+
/** Human-readable env name for the body/title. */
|
|
34
|
+
environmentName?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Build the notification payload for a health-state transition. Pure and
|
|
39
|
+
* side-effect free so it can be unit-tested directly. Extracted from
|
|
40
|
+
* `notifyStateChange` so the body/title/subject wording (which now NAMES the
|
|
41
|
+
* failing check and pushes a `healthcheck.healthcheck` subject) is verifiable
|
|
42
|
+
* without booting the whole queue executor.
|
|
43
|
+
*
|
|
44
|
+
* Recovery bodies stay system-level (the whole system is green again; naming
|
|
45
|
+
* one check would mislead) and omit the check subject. Failing transitions
|
|
46
|
+
* (escalation / de-escalation) name the check in the body and add it as a
|
|
47
|
+
* subject deep-linked to its run history.
|
|
48
|
+
*/
|
|
49
|
+
export function buildHealthTransitionNotification(
|
|
50
|
+
input: HealthTransitionNotificationInput,
|
|
51
|
+
): NotifyForSubscriptionInput {
|
|
52
|
+
const {
|
|
53
|
+
transition,
|
|
54
|
+
systemId,
|
|
55
|
+
systemName,
|
|
56
|
+
configurationId,
|
|
57
|
+
checkName,
|
|
58
|
+
newStatus,
|
|
59
|
+
environmentId,
|
|
60
|
+
environmentName,
|
|
61
|
+
} = input;
|
|
62
|
+
|
|
63
|
+
const envScoped = typeof environmentId === "string";
|
|
64
|
+
const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
|
|
65
|
+
const envQualifier = envScoped
|
|
66
|
+
? ` in environment **${environmentName ?? environmentId}**`
|
|
67
|
+
: "";
|
|
68
|
+
|
|
69
|
+
let title: string;
|
|
70
|
+
let body: string;
|
|
71
|
+
let importance: "info" | "warning" | "critical";
|
|
72
|
+
|
|
73
|
+
if (transition === "recovery") {
|
|
74
|
+
title = `System health restored${envSuffix}: ${systemName}`;
|
|
75
|
+
body = envScoped
|
|
76
|
+
? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
|
|
77
|
+
: `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
|
|
78
|
+
importance = "info";
|
|
79
|
+
} else if (newStatus === "unhealthy") {
|
|
80
|
+
title = `System health critical${envSuffix}: ${systemName}`;
|
|
81
|
+
body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system is unhealthy and may be down${envScoped ? " in that environment" : ""}.`;
|
|
82
|
+
importance = "critical";
|
|
83
|
+
} else {
|
|
84
|
+
// degraded - either an escalation from healthy or a partial recovery
|
|
85
|
+
title = `System health degraded${envSuffix}: ${systemName}`;
|
|
86
|
+
body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system may be experiencing issues${envScoped ? " in that environment" : ""}.`;
|
|
87
|
+
importance = "warning";
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const systemDetailPath = resolveRoute(catalogRoutes.routes.systemDetail, {
|
|
91
|
+
systemId,
|
|
92
|
+
});
|
|
93
|
+
// Recovery lands on the default (all) view; failing transitions deep-link
|
|
94
|
+
// operators into the failing-checks filter so they can debug immediately.
|
|
95
|
+
const actionUrl =
|
|
96
|
+
transition === "recovery"
|
|
97
|
+
? systemDetailPath
|
|
98
|
+
: `${systemDetailPath}?filter=failing`;
|
|
99
|
+
const actionLabel =
|
|
100
|
+
transition === "recovery" ? "View System" : "View failing checks";
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
specId: healthcheckSystemSubscription.specId,
|
|
104
|
+
resourceKeys: [systemId],
|
|
105
|
+
title,
|
|
106
|
+
body,
|
|
107
|
+
importance,
|
|
108
|
+
action: { label: actionLabel, url: actionUrl },
|
|
109
|
+
// Env-qualified collapse key so two failing envs of one system generate
|
|
110
|
+
// two independent notification cards (one per env) instead of merging.
|
|
111
|
+
collapseKey: envScoped
|
|
112
|
+
? systemHealthCollapseKey(systemId, environmentId)
|
|
113
|
+
: systemHealthCollapseKey(systemId),
|
|
114
|
+
subjects: [
|
|
115
|
+
createSystemSubject({
|
|
116
|
+
id: systemId,
|
|
117
|
+
name: systemName,
|
|
118
|
+
url: systemDetailPath,
|
|
119
|
+
status: newStatus,
|
|
120
|
+
}),
|
|
121
|
+
// Name the failing check as its own subject for every non-recovery
|
|
122
|
+
// transition, deep-linked to its run history. Omitted on recovery.
|
|
123
|
+
...(transition === "recovery"
|
|
124
|
+
? []
|
|
125
|
+
: [
|
|
126
|
+
createHealthcheckSubject({
|
|
127
|
+
id: configurationId,
|
|
128
|
+
name: checkName,
|
|
129
|
+
url: resolveRoute(healthcheckRoutes.routes.historyDetail, {
|
|
130
|
+
systemId,
|
|
131
|
+
configurationId,
|
|
132
|
+
}),
|
|
133
|
+
status: newStatus,
|
|
134
|
+
}),
|
|
135
|
+
]),
|
|
136
|
+
],
|
|
137
|
+
};
|
|
138
|
+
}
|
|
@@ -20,6 +20,34 @@ import { Versioned } from "@checkstack/backend-api";
|
|
|
20
20
|
* System → healthchecks extension in isolation with mock services.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
+
// The System extension's reconcile converges the per-env recurring jobs via
|
|
24
|
+
// `reconcileHealthCheckJobs`. That path has its own dedicated tests
|
|
25
|
+
// (schedule-reconciler.test.ts); here we feed it EMPTY db/catalog/queue stubs so
|
|
26
|
+
// it is a safe no-op and these tests can focus on the association logic.
|
|
27
|
+
// (A `mock.module` would leak process-wide and break schedule-reconciler.test.)
|
|
28
|
+
function emptyReconcileDbStub() {
|
|
29
|
+
return {
|
|
30
|
+
select: () => ({
|
|
31
|
+
from: () => ({
|
|
32
|
+
innerJoin: () => ({ where: () => Promise.resolve([]) }),
|
|
33
|
+
groupBy: () => Promise.resolve([]),
|
|
34
|
+
}),
|
|
35
|
+
}),
|
|
36
|
+
} as never;
|
|
37
|
+
}
|
|
38
|
+
function emptyReconcileQueueManager() {
|
|
39
|
+
return {
|
|
40
|
+
getQueue: () => ({
|
|
41
|
+
scheduleRecurring: async () => "job-123",
|
|
42
|
+
listRecurringJobs: async () => [],
|
|
43
|
+
getRecurringJobDetails: async () => undefined,
|
|
44
|
+
cancelRecurring: async () => {},
|
|
45
|
+
}),
|
|
46
|
+
} as never;
|
|
47
|
+
}
|
|
48
|
+
const emptyCatalogClient = () =>
|
|
49
|
+
({ resolveSystemEnvironments: async () => [] }) as never;
|
|
50
|
+
|
|
23
51
|
// ─── Mock Healthcheck Service ──────────────────────────────────────────────
|
|
24
52
|
|
|
25
53
|
interface MockConfig {
|
|
@@ -231,7 +259,9 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
|
|
|
231
259
|
createService: () => mockService as any,
|
|
232
260
|
getHealthCheckRegistry: () => mockHCRegistry as any,
|
|
233
261
|
getCollectorRegistry: () => mockCollectorRegistry as any,
|
|
234
|
-
getQueueManager: () => (
|
|
262
|
+
getQueueManager: () => emptyReconcileQueueManager(),
|
|
263
|
+
getDb: () => emptyReconcileDbStub(),
|
|
264
|
+
getCatalogClient: () => emptyCatalogClient(),
|
|
235
265
|
};
|
|
236
266
|
return buildHealthcheckKind(mockDeps);
|
|
237
267
|
}
|
|
@@ -594,7 +624,9 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
|
|
|
594
624
|
}) as never,
|
|
595
625
|
getHealthCheckRegistry: () => createMockHealthCheckRegistry() as never,
|
|
596
626
|
getCollectorRegistry: () => createMockCollectorRegistry() as never,
|
|
597
|
-
getQueueManager: () => (
|
|
627
|
+
getQueueManager: () => emptyReconcileQueueManager(),
|
|
628
|
+
getDb: () => emptyReconcileDbStub(),
|
|
629
|
+
getCatalogClient: () => emptyCatalogClient(),
|
|
598
630
|
});
|
|
599
631
|
}
|
|
600
632
|
|
|
@@ -25,7 +25,11 @@ import {
|
|
|
25
25
|
enumField,
|
|
26
26
|
} from "@checkstack/backend-api";
|
|
27
27
|
import type { QueueManager } from "@checkstack/queue-api";
|
|
28
|
-
import {
|
|
28
|
+
import type { SafeDatabase } from "@checkstack/backend-api";
|
|
29
|
+
import type { InferClient } from "@checkstack/common";
|
|
30
|
+
import type { CatalogApi } from "@checkstack/catalog-common";
|
|
31
|
+
import type * as schema from "./schema";
|
|
32
|
+
import { reconcileHealthCheckJobs } from "./schedule-reconciler";
|
|
29
33
|
|
|
30
34
|
/**
|
|
31
35
|
* Lazy accessor functions — populated during init(), consumed during reconcile.
|
|
@@ -37,6 +41,8 @@ interface HealthcheckGitOpsKindsDeps {
|
|
|
37
41
|
getHealthCheckRegistry: () => HealthCheckRegistry;
|
|
38
42
|
getCollectorRegistry: () => CollectorRegistry;
|
|
39
43
|
getQueueManager: () => QueueManager;
|
|
44
|
+
getDb: () => SafeDatabase<typeof schema>;
|
|
45
|
+
getCatalogClient: () => InferClient<typeof CatalogApi>;
|
|
40
46
|
}
|
|
41
47
|
|
|
42
48
|
// ─── Healthcheck Spec Schema ───────────────────────────────────────────────
|
|
@@ -382,18 +388,16 @@ export function buildSystemHealthcheckExtension(
|
|
|
382
388
|
notificationPolicy,
|
|
383
389
|
});
|
|
384
390
|
|
|
385
|
-
//
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
});
|
|
396
|
-
}
|
|
391
|
+
// Reconcile this system's per-env recurring jobs so the new
|
|
392
|
+
// association starts probing right away (system-scoped: add/update
|
|
393
|
+
// only, orphan cleanup is owned by the periodic full reconcile).
|
|
394
|
+
await reconcileHealthCheckJobs({
|
|
395
|
+
db: deps.getDb(),
|
|
396
|
+
queueManager: deps.getQueueManager(),
|
|
397
|
+
catalogClient: deps.getCatalogClient(),
|
|
398
|
+
logger: context.logger,
|
|
399
|
+
systemId: systemEntityId,
|
|
400
|
+
});
|
|
397
401
|
|
|
398
402
|
context.logger.info(
|
|
399
403
|
`GitOps: associated ${entry.ref.kind} "${entry.ref.name}" (${configId}) with System "${entity.metadata.name}" and scheduled execution`,
|