@checkstack/healthcheck-backend 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +484 -0
- package/drizzle/0019_chemical_frightful_four.sql +8 -0
- package/drizzle/0020_certain_mordo.sql +2 -0
- package/drizzle/meta/0019_snapshot.json +661 -0
- package/drizzle/meta/0020_snapshot.json +711 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +23 -21
- package/src/ai/system-signals-contributor.test.ts +33 -9
- package/src/ai/system-signals-contributor.ts +38 -16
- package/src/cache-test-stub.ts +26 -0
- package/src/cache.test.ts +291 -0
- package/src/cache.ts +204 -34
- package/src/health-notification-content.test.ts +111 -0
- package/src/health-notification-content.ts +145 -0
- package/src/healthcheck-gitops-kinds.test.ts +14 -0
- package/src/healthcheck-gitops-kinds.ts +27 -0
- package/src/index.ts +31 -12
- package/src/queue-executor.test.ts +13 -26
- package/src/queue-executor.ts +125 -112
- package/src/retention-job.ts +8 -0
- package/src/rollup-consumer.test.ts +19 -8
- package/src/router-config-secrets.test.ts +2 -7
- package/src/router-create-and-assign.test.ts +2 -7
- package/src/router-pause-recompute.test.ts +2 -7
- package/src/router.test.ts +3 -8
- package/src/router.ts +43 -15
- package/src/schema.ts +74 -31
- package/src/service-batching.test.ts +8 -0
- package/src/service-bulk-counts.it.test.ts +144 -0
- package/src/service-bulk-run-stats.it.test.ts +197 -0
- package/src/service-ordering.test.ts +6 -2
- package/src/service-paused-filter.test.ts +13 -0
- package/src/service-rollup-worst-wins.test.ts +209 -145
- package/src/service.ts +408 -284
- package/src/status-fingerprint.test.ts +92 -0
- package/src/status-fingerprint.ts +66 -0
- package/src/status-page/rollup.test.ts +40 -0
- package/src/status-page/rollup.ts +27 -0
- package/src/status-page/widgets.test.ts +387 -0
- package/src/status-page/widgets.ts +236 -39
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test for `HealthCheckService.getBulkRunStats` against a REAL
|
|
3
|
+
* Postgres. The method's whole point is the single grouped read over
|
|
4
|
+
* `health_check_runs` for MANY systems, then per-system summarization - so each
|
|
5
|
+
* entry MUST be byte-identical to what the single `getRunStats({ systemId })`
|
|
6
|
+
* would return for the same window. A mocked db cannot prove the `inArray`
|
|
7
|
+
* grouping / windowing, so this real-DB guard pins the equivalence, the
|
|
8
|
+
* per-system isolation, and the "systems with no runs are omitted" behaviour
|
|
9
|
+
* the status-page uptime column relies on.
|
|
10
|
+
*
|
|
11
|
+
* Gated on CHECKSTACK_IT so it runs in CI (shared compose Postgres) and is
|
|
12
|
+
* skipped in the default `bun test` run, matching the other *.it.test.ts here.
|
|
13
|
+
*/
|
|
14
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test";
|
|
15
|
+
import { drizzle } from "drizzle-orm/node-postgres";
|
|
16
|
+
import { Pool } from "pg";
|
|
17
|
+
import type {
|
|
18
|
+
SafeDatabase,
|
|
19
|
+
HealthCheckRegistry,
|
|
20
|
+
CollectorRegistry,
|
|
21
|
+
} from "@checkstack/backend-api";
|
|
22
|
+
import * as schema from "./schema";
|
|
23
|
+
import { HealthCheckService } from "./service";
|
|
24
|
+
|
|
25
|
+
const PG_URL =
|
|
26
|
+
process.env.CHECKSTACK_IT_PG_URL ??
|
|
27
|
+
"postgres://postgres:postgres@localhost:5432/postgres";
|
|
28
|
+
const SCHEMA = "healthcheck_it_bulk_run_stats";
|
|
29
|
+
|
|
30
|
+
const START = new Date("2026-06-01T00:00:00.000Z");
|
|
31
|
+
const END = new Date("2026-06-01T23:59:59.000Z");
|
|
32
|
+
|
|
33
|
+
let admin: Pool;
|
|
34
|
+
let pool: Pool;
|
|
35
|
+
let service: HealthCheckService;
|
|
36
|
+
|
|
37
|
+
async function insertRun(row: {
|
|
38
|
+
systemId: string;
|
|
39
|
+
status: string;
|
|
40
|
+
latencyMs?: number | null;
|
|
41
|
+
at: string;
|
|
42
|
+
environmentId?: string | null;
|
|
43
|
+
}): Promise<void> {
|
|
44
|
+
await pool.query(
|
|
45
|
+
`INSERT INTO "${SCHEMA}".health_check_runs
|
|
46
|
+
(id, configuration_id, system_id, environment_id, status, latency_ms, timestamp)
|
|
47
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
|
|
48
|
+
[
|
|
49
|
+
crypto.randomUUID(),
|
|
50
|
+
crypto.randomUUID(),
|
|
51
|
+
row.systemId,
|
|
52
|
+
row.environmentId ?? null,
|
|
53
|
+
row.status,
|
|
54
|
+
row.latencyMs ?? null,
|
|
55
|
+
row.at,
|
|
56
|
+
],
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
describe.skipIf(!process.env.CHECKSTACK_IT)(
|
|
61
|
+
"HealthCheckService.getBulkRunStats (shared Postgres)",
|
|
62
|
+
() => {
|
|
63
|
+
beforeAll(async () => {
|
|
64
|
+
admin = new Pool({ connectionString: PG_URL });
|
|
65
|
+
await admin.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
|
|
66
|
+
await admin.query(`CREATE SCHEMA "${SCHEMA}"`);
|
|
67
|
+
// Minimal DDL: only the columns the grouped read touches; status is plain
|
|
68
|
+
// text and configuration_id carries no FK, keeping the schema
|
|
69
|
+
// self-contained like the sibling *.it.test.ts files.
|
|
70
|
+
await admin.query(
|
|
71
|
+
`CREATE TABLE "${SCHEMA}".health_check_runs (
|
|
72
|
+
id uuid PRIMARY KEY,
|
|
73
|
+
configuration_id uuid NOT NULL,
|
|
74
|
+
system_id text NOT NULL,
|
|
75
|
+
environment_id text,
|
|
76
|
+
status text NOT NULL,
|
|
77
|
+
latency_ms integer,
|
|
78
|
+
result jsonb,
|
|
79
|
+
source_id text,
|
|
80
|
+
source_label text,
|
|
81
|
+
timestamp timestamp NOT NULL DEFAULT now()
|
|
82
|
+
)`,
|
|
83
|
+
);
|
|
84
|
+
pool = new Pool({
|
|
85
|
+
connectionString: PG_URL,
|
|
86
|
+
options: `-c search_path=${SCHEMA}`,
|
|
87
|
+
});
|
|
88
|
+
const db = drizzle(pool, {
|
|
89
|
+
schema,
|
|
90
|
+
}) as unknown as SafeDatabase<typeof schema>;
|
|
91
|
+
// registry / collectorRegistry are unused by the stats read under test;
|
|
92
|
+
// stub them so the constructor is satisfied.
|
|
93
|
+
service = new HealthCheckService(
|
|
94
|
+
db,
|
|
95
|
+
{} as unknown as HealthCheckRegistry,
|
|
96
|
+
{} as unknown as CollectorRegistry,
|
|
97
|
+
);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
afterAll(async () => {
|
|
101
|
+
await pool?.end();
|
|
102
|
+
await admin.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
|
|
103
|
+
await admin.end();
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
beforeEach(async () => {
|
|
107
|
+
await pool.query(`TRUNCATE "${SCHEMA}".health_check_runs`);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("returns per-system stats identical to getRunStats for the same window", async () => {
|
|
111
|
+
// sys-a: 3 healthy + 1 unhealthy; sys-b: 2 healthy; sys-c: no runs.
|
|
112
|
+
await insertRun({ systemId: "sys-a", status: "healthy", latencyMs: 10, at: "2026-06-01T01:00:00Z" });
|
|
113
|
+
await insertRun({ systemId: "sys-a", status: "healthy", latencyMs: 20, at: "2026-06-01T02:00:00Z" });
|
|
114
|
+
await insertRun({ systemId: "sys-a", status: "healthy", latencyMs: 30, at: "2026-06-01T03:00:00Z" });
|
|
115
|
+
await insertRun({ systemId: "sys-a", status: "unhealthy", latencyMs: 40, at: "2026-06-01T04:00:00Z" });
|
|
116
|
+
await insertRun({ systemId: "sys-b", status: "healthy", latencyMs: 5, at: "2026-06-01T05:00:00Z" });
|
|
117
|
+
await insertRun({ systemId: "sys-b", status: "healthy", latencyMs: 7, at: "2026-06-01T06:00:00Z" });
|
|
118
|
+
// A run OUTSIDE the window must be ignored by both endpoints.
|
|
119
|
+
await insertRun({ systemId: "sys-a", status: "unhealthy", at: "2026-05-01T00:00:00Z" });
|
|
120
|
+
|
|
121
|
+
const ids = ["sys-a", "sys-b", "sys-c"];
|
|
122
|
+
const stats = await service.getBulkRunStats({
|
|
123
|
+
systemIds: ids,
|
|
124
|
+
startDate: START,
|
|
125
|
+
endDate: END,
|
|
126
|
+
maxBuckets: 24,
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
for (const systemId of ids) {
|
|
130
|
+
const single = await service.getRunStats({ systemId, startDate: START, endDate: END, maxBuckets: 24 });
|
|
131
|
+
if (single.total.runCount === 0) {
|
|
132
|
+
// Zero-run systems are omitted from the bulk record.
|
|
133
|
+
expect(stats[systemId]).toBeUndefined();
|
|
134
|
+
} else {
|
|
135
|
+
expect(stats[systemId]).toEqual(single);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// sys-c had no runs => omitted; only populated systems are keys.
|
|
140
|
+
expect(Object.keys(stats).toSorted()).toEqual(["sys-a", "sys-b"]);
|
|
141
|
+
// Sanity: sys-a uptime = 3/4 healthy = 75%.
|
|
142
|
+
expect(stats["sys-a"]?.total.uptimePct).toBe(75);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("scopes uptime to the selected environments (status-page env filter)", async () => {
|
|
146
|
+
// sys-a: prod 2 healthy; staging 1 healthy + 1 unhealthy; env-less 1
|
|
147
|
+
// unhealthy. A prod-only page must count ONLY the two prod runs (100%),
|
|
148
|
+
// never the staging or env-less runs.
|
|
149
|
+
await insertRun({ systemId: "sys-a", status: "healthy", environmentId: "env-prod", at: "2026-06-01T01:00:00Z" });
|
|
150
|
+
await insertRun({ systemId: "sys-a", status: "healthy", environmentId: "env-prod", at: "2026-06-01T02:00:00Z" });
|
|
151
|
+
await insertRun({ systemId: "sys-a", status: "healthy", environmentId: "env-stage", at: "2026-06-01T03:00:00Z" });
|
|
152
|
+
await insertRun({ systemId: "sys-a", status: "unhealthy", environmentId: "env-stage", at: "2026-06-01T04:00:00Z" });
|
|
153
|
+
await insertRun({ systemId: "sys-a", status: "unhealthy", environmentId: null, at: "2026-06-01T05:00:00Z" });
|
|
154
|
+
|
|
155
|
+
const prodOnly = await service.getBulkRunStats({
|
|
156
|
+
systemIds: ["sys-a"],
|
|
157
|
+
startDate: START,
|
|
158
|
+
endDate: END,
|
|
159
|
+
environmentIds: ["env-prod"],
|
|
160
|
+
maxBuckets: 24,
|
|
161
|
+
});
|
|
162
|
+
expect(prodOnly["sys-a"]?.total.runCount).toBe(2);
|
|
163
|
+
expect(prodOnly["sys-a"]?.total.uptimePct).toBe(100);
|
|
164
|
+
|
|
165
|
+
// No env filter counts every run (5 total, 3 healthy = 60%).
|
|
166
|
+
const all = await service.getBulkRunStats({
|
|
167
|
+
systemIds: ["sys-a"],
|
|
168
|
+
startDate: START,
|
|
169
|
+
endDate: END,
|
|
170
|
+
maxBuckets: 24,
|
|
171
|
+
});
|
|
172
|
+
expect(all["sys-a"]?.total.runCount).toBe(5);
|
|
173
|
+
expect(all["sys-a"]?.total.uptimePct).toBe(60);
|
|
174
|
+
|
|
175
|
+
// getRunStats honors the same set filter for the single-system uptime widget.
|
|
176
|
+
const single = await service.getRunStats({
|
|
177
|
+
systemId: "sys-a",
|
|
178
|
+
startDate: START,
|
|
179
|
+
endDate: END,
|
|
180
|
+
environmentIds: ["env-prod", "env-stage"],
|
|
181
|
+
maxBuckets: 24,
|
|
182
|
+
});
|
|
183
|
+
expect(single.total.runCount).toBe(4);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("returns an empty record for an empty request without querying", async () => {
|
|
187
|
+
expect(
|
|
188
|
+
await service.getBulkRunStats({
|
|
189
|
+
systemIds: [],
|
|
190
|
+
startDate: START,
|
|
191
|
+
endDate: END,
|
|
192
|
+
maxBuckets: 1,
|
|
193
|
+
}),
|
|
194
|
+
).toEqual({});
|
|
195
|
+
});
|
|
196
|
+
},
|
|
197
|
+
);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { describe, it, expect, mock, beforeEach } from "bun:test";
|
|
2
2
|
import type { InferSelectModel } from "drizzle-orm";
|
|
3
|
+
import { withTransactionMock } from "@checkstack/test-utils-backend";
|
|
3
4
|
import { HealthCheckService } from "./service";
|
|
4
5
|
import {
|
|
5
6
|
healthCheckRuns,
|
|
@@ -78,10 +79,13 @@ describe("HealthCheckService data ordering", () => {
|
|
|
78
79
|
orderBy: orderByMock,
|
|
79
80
|
}));
|
|
80
81
|
|
|
81
|
-
|
|
82
|
+
// getSystemHealthOverview batches its reads in ONE scoped transaction, so
|
|
83
|
+
// the mock must expose `.transaction(cb)` (runs `cb` against the same mock
|
|
84
|
+
// db). getHistory/getDetailedHistory run standalone and ignore it.
|
|
85
|
+
return withTransactionMock({
|
|
82
86
|
select: mock(() => ({ from: fromMock })),
|
|
83
87
|
$count: mock(() => Promise.resolve(mockRuns.length)),
|
|
84
|
-
};
|
|
88
|
+
});
|
|
85
89
|
}
|
|
86
90
|
|
|
87
91
|
beforeEach(() => {
|
|
@@ -93,6 +93,12 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
93
93
|
})),
|
|
94
94
|
});
|
|
95
95
|
|
|
96
|
+
// Distinct env keys query (rollup): resolves to no env slices for these
|
|
97
|
+
// fixtures (runs are empty), so the rollup has nothing to evaluate.
|
|
98
|
+
const distinctFrom = Object.assign(Promise.resolve([]), {
|
|
99
|
+
where: mock(() => Promise.resolve([])),
|
|
100
|
+
});
|
|
101
|
+
|
|
96
102
|
let selectCallCount = 0;
|
|
97
103
|
return {
|
|
98
104
|
select: mock(() => {
|
|
@@ -104,6 +110,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
104
110
|
}
|
|
105
111
|
return { from: mock(() => runsFrom) };
|
|
106
112
|
}),
|
|
113
|
+
selectDistinct: mock(() => ({ from: mock(() => distinctFrom) })),
|
|
107
114
|
insert: mock(() => ({
|
|
108
115
|
values: mock(() => ({
|
|
109
116
|
onConflictDoUpdate: mock(() => Promise.resolve()),
|
|
@@ -199,6 +206,11 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
199
206
|
orderBy: runsOrderBy,
|
|
200
207
|
});
|
|
201
208
|
|
|
209
|
+
// Distinct env keys: a single env-less (null) slice for this env-less check.
|
|
210
|
+
const distinctFrom = Object.assign(Promise.resolve([]), {
|
|
211
|
+
where: mock(() => Promise.resolve([{ environmentId: null }])),
|
|
212
|
+
});
|
|
213
|
+
|
|
202
214
|
let selectCallCount = 0;
|
|
203
215
|
const mockDb = {
|
|
204
216
|
select: mock(() => {
|
|
@@ -208,6 +220,7 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
208
220
|
}
|
|
209
221
|
return { from: mock(() => runsFrom) };
|
|
210
222
|
}),
|
|
223
|
+
selectDistinct: mock(() => ({ from: mock(() => distinctFrom) })),
|
|
211
224
|
insert: mock(() => ({
|
|
212
225
|
values: mock(() => ({
|
|
213
226
|
onConflictDoUpdate: mock(() => Promise.resolve()),
|