@checkstack/healthcheck-backend 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +265 -0
- package/package.json +30 -29
- package/src/adaptive-timeout.test.ts +91 -0
- package/src/adaptive-timeout.ts +75 -0
- package/src/ai/system-signals-contributor.test.ts +2 -0
- package/src/automations.test.ts +47 -0
- package/src/automations.ts +19 -3
- package/src/healthcheck-gitops-kinds.test.ts +34 -2
- package/src/healthcheck-gitops-kinds.ts +17 -13
- package/src/index.ts +58 -6
- package/src/migration-chain-contract.test.ts +7 -1
- package/src/notification-policy.test.ts +19 -0
- package/src/notification-policy.ts +26 -0
- package/src/queue-executor.test.ts +391 -338
- package/src/queue-executor.ts +395 -294
- package/src/realtime-aggregation.ts +9 -2
- package/src/rollup-consumer.test.ts +191 -0
- package/src/rollup-consumer.ts +160 -0
- package/src/router.ts +11 -14
- package/src/schedule-jitter.test.ts +69 -0
- package/src/schedule-jitter.ts +50 -0
- package/src/schedule-reconciler.it.test.ts +453 -0
- package/src/schedule-reconciler.test.ts +418 -0
- package/src/schedule-reconciler.ts +304 -0
- package/src/service-batching.test.ts +98 -0
- package/src/service-ordering.test.ts +4 -0
- package/src/service-paused-filter.test.ts +14 -7
- package/src/service-rollup-worst-wins.test.ts +37 -4
- package/src/service.ts +255 -145
- package/src/slow-check-admission.test.ts +184 -0
- package/src/slow-check-admission.ts +101 -0
- package/src/slow-check-classifier.test.ts +155 -0
- package/src/slow-check-classifier.ts +137 -0
- package/src/slow-check-config.ts +102 -0
- package/src/suspect-lane.test.ts +50 -0
- package/src/suspect-lane.ts +61 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { describe, it, expect, mock } from "bun:test";
|
|
2
|
+
import { withTransactionMock } from "@checkstack/test-utils-backend";
|
|
3
|
+
import { HealthCheckService } from "./service";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Regression: `getSystemHealthStatus` batches its 1 (associations) + N
|
|
7
|
+
* (per-check run window) reads into ONE scoped transaction (see
|
|
8
|
+
* `withScopedTransaction`), so the whole read fan-out pays a single
|
|
9
|
+
* BEGIN/SET LOCAL/COMMIT and holds one connection instead of 1+N standalone
|
|
10
|
+
* scoped queries. This pins that the reads run inside `db.transaction(...)`
|
|
11
|
+
* (exactly once per call), independent of how many checks a system has.
|
|
12
|
+
*/
|
|
13
|
+
describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
|
|
14
|
+
function createMockDb(configCount: number) {
|
|
15
|
+
const associations = Array.from({ length: configCount }, (_, i) => ({
|
|
16
|
+
configurationId: `config-${i}`,
|
|
17
|
+
configName: `Check ${i}`,
|
|
18
|
+
enabled: true,
|
|
19
|
+
paused: false,
|
|
20
|
+
stateThresholds: null,
|
|
21
|
+
}));
|
|
22
|
+
const assocWhere = mock(() => Promise.resolve(associations));
|
|
23
|
+
const assocInnerJoin = Object.assign(Promise.resolve([]), {
|
|
24
|
+
where: assocWhere,
|
|
25
|
+
});
|
|
26
|
+
const assocFrom = Object.assign(Promise.resolve([]), {
|
|
27
|
+
innerJoin: mock(() => assocInnerJoin),
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
// Each per-check run window returns one healthy run.
|
|
31
|
+
const healthyRun = [
|
|
32
|
+
{ status: "healthy" as const, timestamp: new Date(), environmentId: null },
|
|
33
|
+
];
|
|
34
|
+
const runsLimit = mock(() => Promise.resolve(healthyRun));
|
|
35
|
+
const runsOrderBy = mock(() => ({ limit: runsLimit }));
|
|
36
|
+
const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
|
|
37
|
+
const runsFrom = Object.assign(Promise.resolve(healthyRun), {
|
|
38
|
+
where: runsWhere,
|
|
39
|
+
orderBy: runsOrderBy,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
let selectCallCount = 0;
|
|
43
|
+
const db = withTransactionMock({
|
|
44
|
+
select: mock(() => {
|
|
45
|
+
selectCallCount += 1;
|
|
46
|
+
if (selectCallCount === 1) return { from: mock(() => assocFrom) };
|
|
47
|
+
return { from: mock(() => runsFrom) };
|
|
48
|
+
}),
|
|
49
|
+
insert: mock(() => ({ values: mock(() => Promise.resolve()) })),
|
|
50
|
+
update: mock(() => ({
|
|
51
|
+
set: mock(() => ({ where: mock(() => Promise.resolve()) })),
|
|
52
|
+
})),
|
|
53
|
+
delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
|
|
54
|
+
execute: mock(() => Promise.resolve()),
|
|
55
|
+
});
|
|
56
|
+
return db;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
it("wraps the associations + per-check reads in exactly ONE transaction", async () => {
|
|
60
|
+
const mockDb = createMockDb(3);
|
|
61
|
+
const service = new HealthCheckService(
|
|
62
|
+
mockDb as never,
|
|
63
|
+
{} as never,
|
|
64
|
+
{} as never,
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
const result = await service.getSystemHealthStatus("system-1");
|
|
68
|
+
|
|
69
|
+
expect(result.status).toBe("healthy");
|
|
70
|
+
expect(result.checkStatuses).toHaveLength(3);
|
|
71
|
+
// One transaction covers all 1 + N reads (not one per query).
|
|
72
|
+
const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
|
|
73
|
+
.transaction;
|
|
74
|
+
expect(transaction).toHaveBeenCalledTimes(1);
|
|
75
|
+
// 1 associations select + 3 per-check run selects = 4 selects, all inside
|
|
76
|
+
// the single transaction.
|
|
77
|
+
const select = (mockDb as unknown as { select: ReturnType<typeof mock> })
|
|
78
|
+
.select;
|
|
79
|
+
expect(select).toHaveBeenCalledTimes(4);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("still opens exactly one transaction for a system with no checks", async () => {
|
|
83
|
+
const mockDb = createMockDb(0);
|
|
84
|
+
const service = new HealthCheckService(
|
|
85
|
+
mockDb as never,
|
|
86
|
+
{} as never,
|
|
87
|
+
{} as never,
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
const result = await service.getSystemHealthStatus("system-1");
|
|
91
|
+
|
|
92
|
+
expect(result.status).toBe("healthy");
|
|
93
|
+
expect(result.checkStatuses).toHaveLength(0);
|
|
94
|
+
const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
|
|
95
|
+
.transaction;
|
|
96
|
+
expect(transaction).toHaveBeenCalledTimes(1);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
@@ -64,6 +64,10 @@ describe("HealthCheckService data ordering", () => {
|
|
|
64
64
|
const whereMock = mock(() => ({
|
|
65
65
|
orderBy: orderByMock,
|
|
66
66
|
limit: mock(createLimitResult),
|
|
67
|
+
// getSystemHealthOverview's "last successful run" query terminates in
|
|
68
|
+
// `.where(...).groupBy(environmentId)` (a max-per-env aggregate). These
|
|
69
|
+
// ordering tests don't assert last-success, so return no groups.
|
|
70
|
+
groupBy: mock(() => Promise.resolve([])),
|
|
67
71
|
}));
|
|
68
72
|
const innerJoinMock = mock(() => ({
|
|
69
73
|
where: mock(() => Promise.resolve([...mockAssociations])),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect, mock, beforeEach } from "bun:test";
|
|
2
|
+
import { withTransactionMock } from "@checkstack/test-utils-backend";
|
|
2
3
|
import { HealthCheckService } from "./service";
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -143,7 +144,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
143
144
|
|
|
144
145
|
const mockDb = createMockDb();
|
|
145
146
|
const service = new HealthCheckService(
|
|
146
|
-
mockDb as never,
|
|
147
|
+
withTransactionMock(mockDb) as never,
|
|
147
148
|
{} as never,
|
|
148
149
|
{} as never,
|
|
149
150
|
);
|
|
@@ -222,7 +223,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
222
223
|
};
|
|
223
224
|
|
|
224
225
|
const service = new HealthCheckService(
|
|
225
|
-
mockDb as never,
|
|
226
|
+
withTransactionMock(mockDb) as never,
|
|
226
227
|
{} as never,
|
|
227
228
|
{} as never,
|
|
228
229
|
);
|
|
@@ -239,7 +240,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
239
240
|
|
|
240
241
|
const mockDb = createMockDb();
|
|
241
242
|
const service = new HealthCheckService(
|
|
242
|
-
mockDb as never,
|
|
243
|
+
withTransactionMock(mockDb) as never,
|
|
243
244
|
{} as never,
|
|
244
245
|
{} as never,
|
|
245
246
|
);
|
|
@@ -265,7 +266,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
265
266
|
};
|
|
266
267
|
|
|
267
268
|
const service = new HealthCheckService(
|
|
268
|
-
mockDb as never,
|
|
269
|
+
withTransactionMock(mockDb) as never,
|
|
269
270
|
{} as never,
|
|
270
271
|
{} as never,
|
|
271
272
|
);
|
|
@@ -285,7 +286,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
285
286
|
};
|
|
286
287
|
|
|
287
288
|
const service = new HealthCheckService(
|
|
288
|
-
mockDb as never,
|
|
289
|
+
withTransactionMock(mockDb) as never,
|
|
289
290
|
{} as never,
|
|
290
291
|
{} as never,
|
|
291
292
|
);
|
|
@@ -340,7 +341,13 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
340
341
|
|
|
341
342
|
const runsLimit = mock(() => Promise.resolve(emptyRuns));
|
|
342
343
|
const runsOrderBy = mock(() => ({ limit: runsLimit }));
|
|
343
|
-
|
|
344
|
+
// getSystemHealthOverview also runs a `.where(...).groupBy(env)` aggregate
|
|
345
|
+
// for the last-successful-run stamp; return no groups here.
|
|
346
|
+
const runsWhere = mock(() => ({
|
|
347
|
+
orderBy: runsOrderBy,
|
|
348
|
+
limit: runsLimit,
|
|
349
|
+
groupBy: mock(() => Promise.resolve([])),
|
|
350
|
+
}));
|
|
344
351
|
const runsFrom = Object.assign(Promise.resolve(emptyRuns), {
|
|
345
352
|
where: runsWhere,
|
|
346
353
|
orderBy: runsOrderBy,
|
|
@@ -370,7 +377,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
370
377
|
};
|
|
371
378
|
|
|
372
379
|
const service = new HealthCheckService(
|
|
373
|
-
mockDb as never,
|
|
380
|
+
withTransactionMock(mockDb) as never,
|
|
374
381
|
{} as never,
|
|
375
382
|
{} as never,
|
|
376
383
|
);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect, mock } from "bun:test";
|
|
2
|
+
import { withTransactionMock } from "@checkstack/test-utils-backend";
|
|
2
3
|
import { HealthCheckService } from "./service";
|
|
3
4
|
import { evaluateHealthStatus } from "./state-evaluator";
|
|
4
5
|
|
|
@@ -65,7 +66,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
|
|
|
65
66
|
});
|
|
66
67
|
|
|
67
68
|
let selectCallCount = 0;
|
|
68
|
-
return {
|
|
69
|
+
return withTransactionMock({
|
|
69
70
|
select: mock(() => {
|
|
70
71
|
selectCallCount += 1;
|
|
71
72
|
if (selectCallCount === 1) return { from: mock(() => assocFrom) };
|
|
@@ -81,7 +82,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
|
|
|
81
82
|
update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
|
|
82
83
|
delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
|
|
83
84
|
execute: mock(() => Promise.resolve()),
|
|
84
|
-
};
|
|
85
|
+
});
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
it("the rollup reports unhealthy when ONE env is permanently unhealthy, the other healthy", async () => {
|
|
@@ -101,6 +102,34 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
|
|
|
101
102
|
expect(result.checkStatuses).toHaveLength(1);
|
|
102
103
|
expect(result.checkStatuses[0].status).toBe("unhealthy");
|
|
103
104
|
expect(result.checkStatuses[0].runsConsidered).toBe(pool.length);
|
|
105
|
+
// Fan-out accounting: two environment slices (prod + staging), one failing.
|
|
106
|
+
expect(result.checkStatuses[0].sliceCount).toBe(2);
|
|
107
|
+
expect(result.checkStatuses[0].failingSliceCount).toBe(1);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("counts every failing environment slice for the fan-out denominator (3 envs, 2 failing)", () => {
|
|
111
|
+
// Three envs of one check: prod + eu unhealthy, staging healthy. The rollup
|
|
112
|
+
// is unhealthy, and the fan-out accounting must report sliceCount 3 with
|
|
113
|
+
// failingSliceCount 2 so the dashboard can render "2 of 3 checks failing".
|
|
114
|
+
const pool: {
|
|
115
|
+
status: "healthy" | "unhealthy";
|
|
116
|
+
timestamp: Date;
|
|
117
|
+
environmentId: string;
|
|
118
|
+
}[] = [];
|
|
119
|
+
for (let i = 0; i < 5; i++) {
|
|
120
|
+
pool.push({ status: "unhealthy", timestamp: new Date(2025, 0, 1, 0, 0, i), environmentId: "prod" });
|
|
121
|
+
pool.push({ status: "unhealthy", timestamp: new Date(2025, 0, 1, 0, 0, i, 250), environmentId: "eu" });
|
|
122
|
+
pool.push({ status: "healthy", timestamp: new Date(2025, 0, 1, 0, 0, i, 500), environmentId: "staging" });
|
|
123
|
+
}
|
|
124
|
+
const runsDesc = pool.toReversed();
|
|
125
|
+
const mockDb = createMockDb(runsDesc as never);
|
|
126
|
+
const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
|
|
127
|
+
|
|
128
|
+
return service.getSystemHealthStatus("system-1").then((result) => {
|
|
129
|
+
expect(result.status).toBe("unhealthy");
|
|
130
|
+
expect(result.checkStatuses[0].sliceCount).toBe(3);
|
|
131
|
+
expect(result.checkStatuses[0].failingSliceCount).toBe(2);
|
|
132
|
+
});
|
|
104
133
|
});
|
|
105
134
|
|
|
106
135
|
it("flattening the same mixed pool through the evaluator (the pre-fix derivation) would have returned `healthy`", async () => {
|
|
@@ -180,7 +209,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
|
|
|
180
209
|
});
|
|
181
210
|
|
|
182
211
|
let selectCallCount = 0;
|
|
183
|
-
const mockDb = {
|
|
212
|
+
const mockDb = withTransactionMock({
|
|
184
213
|
select: mock(() => {
|
|
185
214
|
selectCallCount += 1;
|
|
186
215
|
if (selectCallCount === 1) return { from: mock(() => assocFrom) };
|
|
@@ -196,10 +225,14 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
|
|
|
196
225
|
update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
|
|
197
226
|
delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
|
|
198
227
|
execute: mock(() => Promise.resolve()),
|
|
199
|
-
};
|
|
228
|
+
});
|
|
200
229
|
|
|
201
230
|
const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
|
|
202
231
|
const result = await service.getSystemHealthStatus("system-1", "prod");
|
|
203
232
|
expect(result.status).toBe("unhealthy");
|
|
233
|
+
// A single-env evaluation is always one slice; failing here since prod is
|
|
234
|
+
// unhealthy.
|
|
235
|
+
expect(result.checkStatuses[0].sliceCount).toBe(1);
|
|
236
|
+
expect(result.checkStatuses[0].failingSliceCount).toBe(1);
|
|
204
237
|
});
|
|
205
238
|
});
|