@checkstack/healthcheck-backend 1.16.0 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +399 -0
  2. package/package.json +30 -29
  3. package/src/adaptive-timeout.test.ts +91 -0
  4. package/src/adaptive-timeout.ts +75 -0
  5. package/src/ai/system-signals-contributor.test.ts +2 -0
  6. package/src/automations.test.ts +47 -0
  7. package/src/automations.ts +19 -3
  8. package/src/healthcheck-gitops-kinds.test.ts +34 -2
  9. package/src/healthcheck-gitops-kinds.ts +17 -13
  10. package/src/index.ts +87 -6
  11. package/src/migration-chain-contract.test.ts +7 -1
  12. package/src/notification-policy.test.ts +19 -0
  13. package/src/notification-policy.ts +26 -0
  14. package/src/queue-executor.test.ts +391 -338
  15. package/src/queue-executor.ts +395 -294
  16. package/src/realtime-aggregation.ts +9 -2
  17. package/src/rollup-consumer.test.ts +191 -0
  18. package/src/rollup-consumer.ts +160 -0
  19. package/src/router.ts +103 -19
  20. package/src/schedule-jitter.test.ts +69 -0
  21. package/src/schedule-jitter.ts +50 -0
  22. package/src/schedule-reconciler.it.test.ts +453 -0
  23. package/src/schedule-reconciler.test.ts +418 -0
  24. package/src/schedule-reconciler.ts +304 -0
  25. package/src/service-batching.test.ts +98 -0
  26. package/src/service-ordering.test.ts +4 -0
  27. package/src/service-paused-filter.test.ts +14 -7
  28. package/src/service-rollup-worst-wins.test.ts +37 -4
  29. package/src/service.ts +348 -145
  30. package/src/slow-check-admission.test.ts +184 -0
  31. package/src/slow-check-admission.ts +101 -0
  32. package/src/slow-check-classifier.test.ts +155 -0
  33. package/src/slow-check-classifier.ts +137 -0
  34. package/src/slow-check-config.ts +102 -0
  35. package/src/status-page/widgets.ts +11 -1
  36. package/src/suspect-lane.test.ts +50 -0
  37. package/src/suspect-lane.ts +61 -0
  38. package/src/system-health-override.test.ts +94 -0
  39. package/src/system-health-override.ts +93 -0
@@ -0,0 +1,98 @@
1
+ import { describe, it, expect, mock } from "bun:test";
2
+ import { withTransactionMock } from "@checkstack/test-utils-backend";
3
+ import { HealthCheckService } from "./service";
4
+
5
+ /**
6
+ * Regression: `getSystemHealthStatus` batches its 1 (associations) + N
7
+ * (per-check run window) reads into ONE scoped transaction (see
8
+ * `withScopedTransaction`), so the whole read fan-out pays a single
9
+ * BEGIN/SET LOCAL/COMMIT and holds one connection instead of 1+N standalone
10
+ * scoped queries. This pins that the reads run inside `db.transaction(...)`
11
+ * (exactly once per call), independent of how many checks a system has.
12
+ */
13
+ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
14
+ function createMockDb(configCount: number) {
15
+ const associations = Array.from({ length: configCount }, (_, i) => ({
16
+ configurationId: `config-${i}`,
17
+ configName: `Check ${i}`,
18
+ enabled: true,
19
+ paused: false,
20
+ stateThresholds: null,
21
+ }));
22
+ const assocWhere = mock(() => Promise.resolve(associations));
23
+ const assocInnerJoin = Object.assign(Promise.resolve([]), {
24
+ where: assocWhere,
25
+ });
26
+ const assocFrom = Object.assign(Promise.resolve([]), {
27
+ innerJoin: mock(() => assocInnerJoin),
28
+ });
29
+
30
+ // Each per-check run window returns one healthy run.
31
+ const healthyRun = [
32
+ { status: "healthy" as const, timestamp: new Date(), environmentId: null },
33
+ ];
34
+ const runsLimit = mock(() => Promise.resolve(healthyRun));
35
+ const runsOrderBy = mock(() => ({ limit: runsLimit }));
36
+ const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
37
+ const runsFrom = Object.assign(Promise.resolve(healthyRun), {
38
+ where: runsWhere,
39
+ orderBy: runsOrderBy,
40
+ });
41
+
42
+ let selectCallCount = 0;
43
+ const db = withTransactionMock({
44
+ select: mock(() => {
45
+ selectCallCount += 1;
46
+ if (selectCallCount === 1) return { from: mock(() => assocFrom) };
47
+ return { from: mock(() => runsFrom) };
48
+ }),
49
+ insert: mock(() => ({ values: mock(() => Promise.resolve()) })),
50
+ update: mock(() => ({
51
+ set: mock(() => ({ where: mock(() => Promise.resolve()) })),
52
+ })),
53
+ delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
54
+ execute: mock(() => Promise.resolve()),
55
+ });
56
+ return db;
57
+ }
58
+
59
+ it("wraps the associations + per-check reads in exactly ONE transaction", async () => {
60
+ const mockDb = createMockDb(3);
61
+ const service = new HealthCheckService(
62
+ mockDb as never,
63
+ {} as never,
64
+ {} as never,
65
+ );
66
+
67
+ const result = await service.getSystemHealthStatus("system-1");
68
+
69
+ expect(result.status).toBe("healthy");
70
+ expect(result.checkStatuses).toHaveLength(3);
71
+ // One transaction covers all 1 + N reads (not one per query).
72
+ const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
73
+ .transaction;
74
+ expect(transaction).toHaveBeenCalledTimes(1);
75
+ // 1 associations select + 3 per-check run selects = 4 selects, all inside
76
+ // the single transaction.
77
+ const select = (mockDb as unknown as { select: ReturnType<typeof mock> })
78
+ .select;
79
+ expect(select).toHaveBeenCalledTimes(4);
80
+ });
81
+
82
+ it("still opens exactly one transaction for a system with no checks", async () => {
83
+ const mockDb = createMockDb(0);
84
+ const service = new HealthCheckService(
85
+ mockDb as never,
86
+ {} as never,
87
+ {} as never,
88
+ );
89
+
90
+ const result = await service.getSystemHealthStatus("system-1");
91
+
92
+ expect(result.status).toBe("healthy");
93
+ expect(result.checkStatuses).toHaveLength(0);
94
+ const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
95
+ .transaction;
96
+ expect(transaction).toHaveBeenCalledTimes(1);
97
+ });
98
+ });
@@ -64,6 +64,10 @@ describe("HealthCheckService data ordering", () => {
64
64
  const whereMock = mock(() => ({
65
65
  orderBy: orderByMock,
66
66
  limit: mock(createLimitResult),
67
+ // getSystemHealthOverview's "last successful run" query terminates in
68
+ // `.where(...).groupBy(environmentId)` (a max-per-env aggregate). These
69
+ // ordering tests don't assert last-success, so return no groups.
70
+ groupBy: mock(() => Promise.resolve([])),
67
71
  }));
68
72
  const innerJoinMock = mock(() => ({
69
73
  where: mock(() => Promise.resolve([...mockAssociations])),
@@ -1,4 +1,5 @@
1
1
  import { describe, it, expect, mock, beforeEach } from "bun:test";
2
+ import { withTransactionMock } from "@checkstack/test-utils-backend";
2
3
  import { HealthCheckService } from "./service";
3
4
 
4
5
  /**
@@ -143,7 +144,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
143
144
 
144
145
  const mockDb = createMockDb();
145
146
  const service = new HealthCheckService(
146
- mockDb as never,
147
+ withTransactionMock(mockDb) as never,
147
148
  {} as never,
148
149
  {} as never,
149
150
  );
@@ -222,7 +223,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
222
223
  };
223
224
 
224
225
  const service = new HealthCheckService(
225
- mockDb as never,
226
+ withTransactionMock(mockDb) as never,
226
227
  {} as never,
227
228
  {} as never,
228
229
  );
@@ -239,7 +240,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
239
240
 
240
241
  const mockDb = createMockDb();
241
242
  const service = new HealthCheckService(
242
- mockDb as never,
243
+ withTransactionMock(mockDb) as never,
243
244
  {} as never,
244
245
  {} as never,
245
246
  );
@@ -265,7 +266,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
265
266
  };
266
267
 
267
268
  const service = new HealthCheckService(
268
- mockDb as never,
269
+ withTransactionMock(mockDb) as never,
269
270
  {} as never,
270
271
  {} as never,
271
272
  );
@@ -285,7 +286,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
285
286
  };
286
287
 
287
288
  const service = new HealthCheckService(
288
- mockDb as never,
289
+ withTransactionMock(mockDb) as never,
289
290
  {} as never,
290
291
  {} as never,
291
292
  );
@@ -340,7 +341,13 @@ describe("HealthCheckService - paused configuration filtering", () => {
340
341
 
341
342
  const runsLimit = mock(() => Promise.resolve(emptyRuns));
342
343
  const runsOrderBy = mock(() => ({ limit: runsLimit }));
343
- const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
344
+ // getSystemHealthOverview also runs a `.where(...).groupBy(env)` aggregate
345
+ // for the last-successful-run stamp; return no groups here.
346
+ const runsWhere = mock(() => ({
347
+ orderBy: runsOrderBy,
348
+ limit: runsLimit,
349
+ groupBy: mock(() => Promise.resolve([])),
350
+ }));
344
351
  const runsFrom = Object.assign(Promise.resolve(emptyRuns), {
345
352
  where: runsWhere,
346
353
  orderBy: runsOrderBy,
@@ -370,7 +377,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
370
377
  };
371
378
 
372
379
  const service = new HealthCheckService(
373
- mockDb as never,
380
+ withTransactionMock(mockDb) as never,
374
381
  {} as never,
375
382
  {} as never,
376
383
  );
@@ -1,4 +1,5 @@
1
1
  import { describe, it, expect, mock } from "bun:test";
2
+ import { withTransactionMock } from "@checkstack/test-utils-backend";
2
3
  import { HealthCheckService } from "./service";
3
4
  import { evaluateHealthStatus } from "./state-evaluator";
4
5
 
@@ -65,7 +66,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
65
66
  });
66
67
 
67
68
  let selectCallCount = 0;
68
- return {
69
+ return withTransactionMock({
69
70
  select: mock(() => {
70
71
  selectCallCount += 1;
71
72
  if (selectCallCount === 1) return { from: mock(() => assocFrom) };
@@ -81,7 +82,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
81
82
  update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
82
83
  delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
83
84
  execute: mock(() => Promise.resolve()),
84
- };
85
+ });
85
86
  }
86
87
 
87
88
  it("the rollup reports unhealthy when ONE env is permanently unhealthy, the other healthy", async () => {
@@ -101,6 +102,34 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
101
102
  expect(result.checkStatuses).toHaveLength(1);
102
103
  expect(result.checkStatuses[0].status).toBe("unhealthy");
103
104
  expect(result.checkStatuses[0].runsConsidered).toBe(pool.length);
105
+ // Fan-out accounting: two environment slices (prod + staging), one failing.
106
+ expect(result.checkStatuses[0].sliceCount).toBe(2);
107
+ expect(result.checkStatuses[0].failingSliceCount).toBe(1);
108
+ });
109
+
110
+ it("counts every failing environment slice for the fan-out denominator (3 envs, 2 failing)", () => {
111
+ // Three envs of one check: prod + eu unhealthy, staging healthy. The rollup
112
+ // is unhealthy, and the fan-out accounting must report sliceCount 3 with
113
+ // failingSliceCount 2 so the dashboard can render "2 of 3 checks failing".
114
+ const pool: {
115
+ status: "healthy" | "unhealthy";
116
+ timestamp: Date;
117
+ environmentId: string;
118
+ }[] = [];
119
+ for (let i = 0; i < 5; i++) {
120
+ pool.push({ status: "unhealthy", timestamp: new Date(2025, 0, 1, 0, 0, i), environmentId: "prod" });
121
+ pool.push({ status: "unhealthy", timestamp: new Date(2025, 0, 1, 0, 0, i, 250), environmentId: "eu" });
122
+ pool.push({ status: "healthy", timestamp: new Date(2025, 0, 1, 0, 0, i, 500), environmentId: "staging" });
123
+ }
124
+ const runsDesc = pool.toReversed();
125
+ const mockDb = createMockDb(runsDesc as never);
126
+ const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
127
+
128
+ return service.getSystemHealthStatus("system-1").then((result) => {
129
+ expect(result.status).toBe("unhealthy");
130
+ expect(result.checkStatuses[0].sliceCount).toBe(3);
131
+ expect(result.checkStatuses[0].failingSliceCount).toBe(2);
132
+ });
104
133
  });
105
134
 
106
135
  it("flattening the same mixed pool through the evaluator (the pre-fix derivation) would have returned `healthy`", async () => {
@@ -180,7 +209,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
180
209
  });
181
210
 
182
211
  let selectCallCount = 0;
183
- const mockDb = {
212
+ const mockDb = withTransactionMock({
184
213
  select: mock(() => {
185
214
  selectCallCount += 1;
186
215
  if (selectCallCount === 1) return { from: mock(() => assocFrom) };
@@ -196,10 +225,14 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
196
225
  update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
197
226
  delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
198
227
  execute: mock(() => Promise.resolve()),
199
- };
228
+ });
200
229
 
201
230
  const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
202
231
  const result = await service.getSystemHealthStatus("system-1", "prod");
203
232
  expect(result.status).toBe("unhealthy");
233
+ // A single-env evaluation is always one slice; failing here since prod is
234
+ // unhealthy.
235
+ expect(result.checkStatuses[0].sliceCount).toBe(1);
236
+ expect(result.checkStatuses[0].failingSliceCount).toBe(1);
204
237
  });
205
238
  });