@checkstack/healthcheck-backend 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/drizzle/0019_chemical_frightful_four.sql +8 -0
- package/drizzle/0020_certain_mordo.sql +2 -0
- package/drizzle/meta/0019_snapshot.json +661 -0
- package/drizzle/meta/0020_snapshot.json +711 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -21
- package/src/ai/system-signals-contributor.test.ts +33 -9
- package/src/ai/system-signals-contributor.ts +38 -16
- package/src/cache-test-stub.ts +26 -0
- package/src/cache.test.ts +291 -0
- package/src/cache.ts +204 -34
- package/src/health-notification-content.test.ts +22 -0
- package/src/health-notification-content.ts +7 -0
- package/src/healthcheck-gitops-kinds.test.ts +14 -0
- package/src/healthcheck-gitops-kinds.ts +27 -0
- package/src/index.ts +31 -12
- package/src/queue-executor.test.ts +13 -26
- package/src/queue-executor.ts +94 -44
- package/src/retention-job.ts +8 -0
- package/src/rollup-consumer.test.ts +19 -8
- package/src/router-config-secrets.test.ts +2 -7
- package/src/router-create-and-assign.test.ts +2 -7
- package/src/router-pause-recompute.test.ts +2 -7
- package/src/router.test.ts +3 -8
- package/src/router.ts +7 -15
- package/src/schema.ts +74 -31
- package/src/service.ts +45 -102
- package/src/status-fingerprint.test.ts +92 -0
- package/src/status-fingerprint.ts +66 -0
- package/src/status-page/widgets.test.ts +84 -0
- package/src/status-page/widgets.ts +87 -6
package/src/queue-executor.ts
CHANGED
|
@@ -379,18 +379,20 @@ export async function recomputeSystemRollupHealth(args: {
|
|
|
379
379
|
// pure entity-recompute path (pause/resume) skips the extra prev read.
|
|
380
380
|
const wantsSignal = signalService !== undefined || cache !== undefined;
|
|
381
381
|
try {
|
|
382
|
-
|
|
382
|
+
// Capture the FULL rollup states (not just the status enum) so the cache
|
|
383
|
+
// reconcile can gate on the per-check vector while the frontend signal
|
|
384
|
+
// stays gated on the coarser rollup-enum transition.
|
|
385
|
+
let previousState: AggregatedHealth | undefined;
|
|
383
386
|
if (wantsSignal) {
|
|
384
|
-
|
|
385
|
-
previousStatus = previousState.status;
|
|
387
|
+
previousState = await service.getSystemHealthStatus(systemId);
|
|
386
388
|
}
|
|
387
|
-
let
|
|
389
|
+
let newState: AggregatedHealth | undefined = previousState;
|
|
388
390
|
await writeHealthEntity({
|
|
389
391
|
handle: getHealthEntity?.(),
|
|
390
392
|
entityId: rollupEntityId,
|
|
391
393
|
apply: async () => {
|
|
392
394
|
const rollupState = await service.getSystemHealthStatus(systemId);
|
|
393
|
-
|
|
395
|
+
newState = rollupState;
|
|
394
396
|
return toHealthEntityView(rollupState);
|
|
395
397
|
},
|
|
396
398
|
serialize: makeHealthSerializer(rollupEntityId),
|
|
@@ -401,21 +403,23 @@ export async function recomputeSystemRollupHealth(args: {
|
|
|
401
403
|
),
|
|
402
404
|
});
|
|
403
405
|
|
|
404
|
-
if (
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
406
|
+
if (wantsSignal && previousState !== undefined && newState !== undefined) {
|
|
407
|
+
// Cache: evict the rollup key + broadcast to the cluster on ANY per-check
|
|
408
|
+
// vector change — a check that flips while the rollup enum stays put still
|
|
409
|
+
// changes the rollup's `checkStatuses`, and a reader gets that vector.
|
|
410
|
+
await cache?.reconcile({ systemId, previous: previousState, next: newState });
|
|
411
|
+
// Frontend signal: only a rollup-enum transition moves the badge, so a
|
|
412
|
+
// per-check-only change needs no SYSTEM_STATUS_CHANGED refetch signal.
|
|
413
|
+
if (newState.status !== previousState.status) {
|
|
414
|
+
await signalService?.broadcast(SYSTEM_STATUS_CHANGED, {
|
|
415
|
+
systemId,
|
|
416
|
+
previousStatus: previousState.status,
|
|
417
|
+
newStatus: newState.status,
|
|
418
|
+
});
|
|
419
|
+
}
|
|
416
420
|
}
|
|
417
|
-
return
|
|
418
|
-
? { previousStatus, newStatus }
|
|
421
|
+
return previousState !== undefined && newState !== undefined
|
|
422
|
+
? { previousStatus: previousState.status, newStatus: newState.status }
|
|
419
423
|
: undefined;
|
|
420
424
|
} catch (error) {
|
|
421
425
|
// A recompute failure must never break the pause/resume RPC. The
|
|
@@ -692,13 +696,14 @@ async function executeHealthCheckJob(props: {
|
|
|
692
696
|
// other.
|
|
693
697
|
const makeHealthSerializer = createHealthEntitySerializer({ advisoryLock });
|
|
694
698
|
|
|
695
|
-
//
|
|
696
|
-
//
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
// the
|
|
700
|
-
|
|
701
|
-
|
|
699
|
+
// NOTE: the system-rollup status BEFORE this tick is computed LAZILY, only on
|
|
700
|
+
// the catastrophic-failure path that actually consumes it (see the `catch`
|
|
701
|
+
// below). It used to be captured here on EVERY run - a full worst-wins rollup
|
|
702
|
+
// (`getSystemHealthStatus(systemId)`, an N+1 across every check × environment)
|
|
703
|
+
// - even though the normal success/failure paths record their transition from
|
|
704
|
+
// the per-env pre-read (`previousState`, below) and never touch the rollup
|
|
705
|
+
// pre-state. Deferring it to the rare error path removes that whole recompute
|
|
706
|
+
// from the hot path of every check tick.
|
|
702
707
|
|
|
703
708
|
// Slow-check lane admission (set when this run was admitted to the suspect
|
|
704
709
|
// lane); released in the outer finally so the slot frees on any exit path.
|
|
@@ -954,14 +959,16 @@ async function executeHealthCheckJob(props: {
|
|
|
954
959
|
const envEntityId = encodeHealthEntityId({ systemId, environmentId });
|
|
955
960
|
const serializeEnvWrite = makeHealthSerializer(envEntityId);
|
|
956
961
|
|
|
957
|
-
// Per-env baseline
|
|
958
|
-
//
|
|
959
|
-
//
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
962
|
+
// Per-env baseline: the env-scoped aggregate BEFORE this run. Read INSIDE
|
|
963
|
+
// the serialized `apply` below (assigned to these vars), NOT here — so a
|
|
964
|
+
// concurrent same-slice run cannot commit between the baseline read and
|
|
965
|
+
// our own insert. If it were read here (outside the `health:<envEntityId>`
|
|
966
|
+
// lock), the cache change-gate could compare `next` against a baseline a
|
|
967
|
+
// sibling run already superseded and miss a real transition, stranding a
|
|
968
|
+
// stale cached status until the TTL. Assigned by whichever branch's
|
|
969
|
+
// `apply` runs; used for the transition log AND the cache reconcile.
|
|
970
|
+
let previousState!: AggregatedHealth;
|
|
971
|
+
let previousStatus!: HealthCheckStatus;
|
|
965
972
|
|
|
966
973
|
// Curated, read-only run-context metadata exposed to collectors.
|
|
967
974
|
// Metadata only - never secrets or config. `environment` carries the
|
|
@@ -1269,6 +1276,13 @@ async function executeHealthCheckJob(props: {
|
|
|
1269
1276
|
handle: getHealthEntity?.(),
|
|
1270
1277
|
entityId: envEntityId,
|
|
1271
1278
|
apply: async () => {
|
|
1279
|
+
// In-lock pre-run baseline (see the `previousState` declaration): read
|
|
1280
|
+
// here, inside the serialized critical section, before the insert.
|
|
1281
|
+
previousState = await service.getSystemHealthStatus(
|
|
1282
|
+
systemId,
|
|
1283
|
+
environmentId,
|
|
1284
|
+
);
|
|
1285
|
+
previousStatus = previousState.status;
|
|
1272
1286
|
// §perf: batch the run INSERT + aggregate SELECT/UPSERT under ONE
|
|
1273
1287
|
// `SET LOCAL search_path` transaction (3 scoped-db transactions → 1),
|
|
1274
1288
|
// which also makes the run and its aggregate commit atomically.
|
|
@@ -1316,9 +1330,18 @@ async function executeHealthCheckJob(props: {
|
|
|
1316
1330
|
`Health check ${configId} for system ${systemId} failed: ${finalError}`,
|
|
1317
1331
|
);
|
|
1318
1332
|
|
|
1319
|
-
//
|
|
1320
|
-
//
|
|
1321
|
-
|
|
1333
|
+
// Reconcile this environment's cached status: evict + broadcast to the
|
|
1334
|
+
// cluster ONLY when the per-check vector actually changed (a run that
|
|
1335
|
+
// leaves every check's status unchanged keeps the cache warm instead of
|
|
1336
|
+
// thrashing it every tick). The rollup key is reconciled separately by
|
|
1337
|
+
// the debounced rollup consumer (recomputeSystemRollupHealth), also
|
|
1338
|
+
// vector-gated.
|
|
1339
|
+
await cache.reconcile({
|
|
1340
|
+
systemId,
|
|
1341
|
+
environmentId,
|
|
1342
|
+
previous: previousState,
|
|
1343
|
+
next: newState,
|
|
1344
|
+
});
|
|
1322
1345
|
|
|
1323
1346
|
await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
|
|
1324
1347
|
systemId,
|
|
@@ -1427,6 +1450,13 @@ async function executeHealthCheckJob(props: {
|
|
|
1427
1450
|
handle: getHealthEntity?.(),
|
|
1428
1451
|
entityId: envEntityId,
|
|
1429
1452
|
apply: async () => {
|
|
1453
|
+
// In-lock pre-run baseline (see the `previousState` declaration): read
|
|
1454
|
+
// here, inside the serialized critical section, before the insert.
|
|
1455
|
+
previousState = await service.getSystemHealthStatus(
|
|
1456
|
+
systemId,
|
|
1457
|
+
environmentId,
|
|
1458
|
+
);
|
|
1459
|
+
previousStatus = previousState.status;
|
|
1430
1460
|
// §perf: batch the run INSERT + aggregate SELECT/UPSERT under ONE
|
|
1431
1461
|
// `SET LOCAL search_path` transaction (3 scoped-db transactions → 1),
|
|
1432
1462
|
// which also makes the run and its aggregate commit atomically.
|
|
@@ -1473,9 +1503,16 @@ async function executeHealthCheckJob(props: {
|
|
|
1473
1503
|
`Ran health check ${configId} for system ${systemId}: ${result.status}`,
|
|
1474
1504
|
);
|
|
1475
1505
|
|
|
1476
|
-
//
|
|
1477
|
-
//
|
|
1478
|
-
|
|
1506
|
+
// Reconcile this environment's cached status: evict + broadcast to the
|
|
1507
|
+
// cluster ONLY when the per-check vector actually changed (a steady-state
|
|
1508
|
+
// healthy run keeps the cache warm). The rollup key is reconciled by the
|
|
1509
|
+
// debounced rollup consumer (recomputeSystemRollupHealth), also vector-gated.
|
|
1510
|
+
await cache.reconcile({
|
|
1511
|
+
systemId,
|
|
1512
|
+
environmentId,
|
|
1513
|
+
previous: previousState,
|
|
1514
|
+
next: newState,
|
|
1515
|
+
});
|
|
1479
1516
|
|
|
1480
1517
|
// Broadcast enriched signal for realtime frontend updates (e.g., terminal feed)
|
|
1481
1518
|
await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
|
|
@@ -1582,12 +1619,19 @@ async function executeHealthCheckJob(props: {
|
|
|
1582
1619
|
// the system-level health change still emits. Reuses the pre-tick
|
|
1583
1620
|
// rollup status captured before the try block.
|
|
1584
1621
|
const rollupEntityId = encodeHealthEntityId({ systemId });
|
|
1585
|
-
|
|
1622
|
+
// The pre-failure rollup baseline. Read INSIDE `apply` (inside the
|
|
1623
|
+
// `health:<systemId>` lock), before the failure-run insert, so a concurrent
|
|
1624
|
+
// catastrophic tick for the same system can't commit between the baseline
|
|
1625
|
+
// read and this insert and make the cache change-gate miss a transition.
|
|
1626
|
+
let rollupPreState!: AggregatedHealth;
|
|
1627
|
+
let previousStatus!: HealthCheckStatus;
|
|
1586
1628
|
let newState!: AggregatedHealth;
|
|
1587
1629
|
await writeHealthEntity({
|
|
1588
1630
|
handle: getHealthEntity?.(),
|
|
1589
1631
|
entityId: rollupEntityId,
|
|
1590
1632
|
apply: async () => {
|
|
1633
|
+
rollupPreState = await service.getSystemHealthStatus(systemId);
|
|
1634
|
+
previousStatus = rollupPreState.status;
|
|
1591
1635
|
// §perf: batch the failure run INSERT + aggregate SELECT/UPSERT under
|
|
1592
1636
|
// ONE `SET LOCAL search_path` transaction (3 scoped-db transactions →
|
|
1593
1637
|
// 1), which also makes them commit atomically.
|
|
@@ -1646,9 +1690,15 @@ async function executeHealthCheckJob(props: {
|
|
|
1646
1690
|
// Use IDs as fallback
|
|
1647
1691
|
}
|
|
1648
1692
|
|
|
1649
|
-
//
|
|
1650
|
-
//
|
|
1651
|
-
|
|
1693
|
+
// Reconcile the rollup cache: evict + broadcast only on a real vector
|
|
1694
|
+
// change. This catastrophic path writes the bare `<systemId>` entity (it IS
|
|
1695
|
+
// the rollup), so it owns the rollup key directly — no debounced consumer
|
|
1696
|
+
// runs for it.
|
|
1697
|
+
await cache.reconcile({
|
|
1698
|
+
systemId,
|
|
1699
|
+
previous: rollupPreState,
|
|
1700
|
+
next: newState,
|
|
1701
|
+
});
|
|
1652
1702
|
|
|
1653
1703
|
// Broadcast enriched failure signal for realtime frontend updates
|
|
1654
1704
|
await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
|
package/src/retention-job.ts
CHANGED
|
@@ -153,6 +153,14 @@ async function deleteExpiredRawRuns(params: DeleteExpiredRawRunsParams) {
|
|
|
153
153
|
const cutoffDate = new Date();
|
|
154
154
|
cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
|
|
155
155
|
|
|
156
|
+
// Status-cache note: this delete does NOT invalidate the system-health status
|
|
157
|
+
// cache, and deliberately need not. `evaluateHealthStatus` derives status from
|
|
158
|
+
// the most-recent-N runs (count-based, no time component), and the cutoff is
|
|
159
|
+
// days old, so for any ACTIVELY-running check the deleted rows are already
|
|
160
|
+
// outside the evaluation window — removing them cannot change the current
|
|
161
|
+
// derived status. The only case it could is a check that STOPPED running long
|
|
162
|
+
// enough for its entire history to age past the cutoff; that flip is bounded by
|
|
163
|
+
// the 15s status-cache TTL (an acceptable, rare edge for a stopped check).
|
|
156
164
|
await db
|
|
157
165
|
.delete(healthCheckRuns)
|
|
158
166
|
.where(
|
|
@@ -35,7 +35,7 @@ interface Harness {
|
|
|
35
35
|
changeHandler: (change: EntityChanged) => Promise<void>;
|
|
36
36
|
getSystemHealthStatus: ReturnType<typeof mock>;
|
|
37
37
|
broadcast: ReturnType<typeof mock>;
|
|
38
|
-
|
|
38
|
+
reconcile: ReturnType<typeof mock>;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
async function setup(opts: {
|
|
@@ -75,7 +75,7 @@ async function setup(opts: {
|
|
|
75
75
|
}) as unknown as OnEntityChanged;
|
|
76
76
|
|
|
77
77
|
const broadcast = mock(async () => {});
|
|
78
|
-
const
|
|
78
|
+
const reconcile = mock(async () => {});
|
|
79
79
|
|
|
80
80
|
await setupRollupConsumer({
|
|
81
81
|
queueManager,
|
|
@@ -85,7 +85,7 @@ async function setup(opts: {
|
|
|
85
85
|
withXactLock: async ({ fn }: { fn: () => Promise<unknown> }) => fn(),
|
|
86
86
|
} as never,
|
|
87
87
|
signalService: { broadcast } as never,
|
|
88
|
-
cache: {
|
|
88
|
+
cache: { reconcile } as never,
|
|
89
89
|
// No entity handle: writeHealthEntity runs `apply` directly.
|
|
90
90
|
getHealthEntity: () => undefined,
|
|
91
91
|
logger: {
|
|
@@ -103,7 +103,7 @@ async function setup(opts: {
|
|
|
103
103
|
changeHandler,
|
|
104
104
|
getSystemHealthStatus,
|
|
105
105
|
broadcast,
|
|
106
|
-
|
|
106
|
+
reconcile,
|
|
107
107
|
};
|
|
108
108
|
}
|
|
109
109
|
|
|
@@ -162,12 +162,21 @@ describe("setupRollupConsumer subscription", () => {
|
|
|
162
162
|
});
|
|
163
163
|
|
|
164
164
|
describe("setupRollupConsumer rollup recompute", () => {
|
|
165
|
-
it("
|
|
165
|
+
it("reconciles the cache + broadcasts SYSTEM_STATUS_CHANGED on a rollup status change", async () => {
|
|
166
166
|
const h = await setup({ statuses: ["healthy", "unhealthy"] });
|
|
167
167
|
await h.consumeHandler({ data: { systemId: "s1" } });
|
|
168
168
|
|
|
169
169
|
expect(h.getSystemHealthStatus).toHaveBeenCalled();
|
|
170
|
-
|
|
170
|
+
// Reconcile is handed the FULL prev/next states; the change-gating (evict +
|
|
171
|
+
// cluster broadcast only on a per-check vector change) lives inside the cache
|
|
172
|
+
// (see cache.test.ts), so recompute always calls it with both states.
|
|
173
|
+
expect(h.reconcile).toHaveBeenCalledTimes(1);
|
|
174
|
+
expect(h.reconcile.mock.calls[0]![0]).toMatchObject({
|
|
175
|
+
systemId: "s1",
|
|
176
|
+
previous: { status: "healthy" },
|
|
177
|
+
next: { status: "unhealthy" },
|
|
178
|
+
});
|
|
179
|
+
// The frontend signal IS gated here on the rollup-enum transition.
|
|
171
180
|
expect(h.broadcast).toHaveBeenCalledTimes(1);
|
|
172
181
|
const payload = h.broadcast.mock.calls[0]![1] as {
|
|
173
182
|
systemId: string;
|
|
@@ -181,11 +190,13 @@ describe("setupRollupConsumer rollup recompute", () => {
|
|
|
181
190
|
});
|
|
182
191
|
});
|
|
183
192
|
|
|
184
|
-
it("does NOT broadcast when the rollup status is unchanged", async () => {
|
|
193
|
+
it("does NOT broadcast the frontend signal when the rollup status is unchanged", async () => {
|
|
185
194
|
const h = await setup({ statuses: ["degraded", "degraded"] });
|
|
186
195
|
await h.consumeHandler({ data: { systemId: "s1" } });
|
|
187
196
|
|
|
197
|
+
// No enum transition ⇒ no SYSTEM_STATUS_CHANGED frontend signal. Reconcile is
|
|
198
|
+
// still invoked (its own fingerprint gate no-ops for an unchanged vector).
|
|
188
199
|
expect(h.broadcast).not.toHaveBeenCalled();
|
|
189
|
-
expect(h.
|
|
200
|
+
expect(h.reconcile).toHaveBeenCalledTimes(1);
|
|
190
201
|
});
|
|
191
202
|
});
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
healthcheckSecretMarker,
|
|
17
17
|
isHealthcheckSecretMarker,
|
|
18
18
|
} from "./config-secrets";
|
|
19
|
-
import
|
|
19
|
+
import { createStubHealthCheckCache } from "./cache-test-stub";
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Guards the SEC-1 fix: `createAndAssign` (the first-check wizard / AI propose
|
|
@@ -24,12 +24,7 @@ import type { HealthCheckCache } from "./cache";
|
|
|
24
24
|
* and return a REDACTED config - never persist or echo a plaintext credential.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
const passthroughCache
|
|
28
|
-
wrapSystemHealthStatus: (_systemId, loader) => loader(),
|
|
29
|
-
invalidateSystem: async () => {},
|
|
30
|
-
invalidateAllSystems: async () => 0,
|
|
31
|
-
scope: {} as HealthCheckCache["scope"],
|
|
32
|
-
};
|
|
27
|
+
const passthroughCache = createStubHealthCheckCache();
|
|
33
28
|
|
|
34
29
|
const mockUser = {
|
|
35
30
|
type: "user" as const,
|
|
@@ -2,7 +2,7 @@ import { describe, it, expect, mock } from "bun:test";
|
|
|
2
2
|
import { createHealthCheckRouter } from "./router";
|
|
3
3
|
import { createMockRpcContext } from "@checkstack/backend-api";
|
|
4
4
|
import { call } from "@orpc/server";
|
|
5
|
-
import
|
|
5
|
+
import { createStubHealthCheckCache } from "./cache-test-stub";
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Router-level tests for the atomic create+assign path and the
|
|
@@ -12,12 +12,7 @@ import type { HealthCheckCache } from "./cache";
|
|
|
12
12
|
* `input.body.notificationPolicy`).
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
const passthroughCache
|
|
16
|
-
wrapSystemHealthStatus: (_systemId, loader) => loader(),
|
|
17
|
-
invalidateSystem: async () => {},
|
|
18
|
-
invalidateAllSystems: async () => 0,
|
|
19
|
-
scope: {} as HealthCheckCache["scope"],
|
|
20
|
-
};
|
|
15
|
+
const passthroughCache = createStubHealthCheckCache();
|
|
21
16
|
|
|
22
17
|
const mockUser = {
|
|
23
18
|
type: "user" as const,
|
|
@@ -2,7 +2,7 @@ import { describe, it, expect, mock, beforeEach } from "bun:test";
|
|
|
2
2
|
import { createHealthCheckRouter } from "./router";
|
|
3
3
|
import { createMockRpcContext } from "@checkstack/backend-api";
|
|
4
4
|
import { call } from "@orpc/server";
|
|
5
|
-
import
|
|
5
|
+
import { createStubHealthCheckCache } from "./cache-test-stub";
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Router-level tests for the pause/resume handlers' rollup-health recompute
|
|
@@ -14,12 +14,7 @@ import type { HealthCheckCache } from "./cache";
|
|
|
14
14
|
* any degraded transition (see the resume handler comment for rationale).
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
const passthroughCache
|
|
18
|
-
wrapSystemHealthStatus: (_systemId, loader) => loader(),
|
|
19
|
-
invalidateSystem: async () => {},
|
|
20
|
-
invalidateAllSystems: async () => 0,
|
|
21
|
-
scope: {} as HealthCheckCache["scope"],
|
|
22
|
-
};
|
|
17
|
+
const passthroughCache = createStubHealthCheckCache();
|
|
23
18
|
|
|
24
19
|
const mockUser = {
|
|
25
20
|
type: "user" as const,
|
package/src/router.test.ts
CHANGED
|
@@ -3,14 +3,9 @@ import { createHealthCheckRouter } from "./router";
|
|
|
3
3
|
import { createMockRpcContext, Versioned } from "@checkstack/backend-api";
|
|
4
4
|
import { call } from "@orpc/server";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
const passthroughCache
|
|
9
|
-
wrapSystemHealthStatus: (_systemId, loader) => loader(),
|
|
10
|
-
invalidateSystem: async () => {},
|
|
11
|
-
invalidateAllSystems: async () => 0,
|
|
12
|
-
scope: {} as HealthCheckCache["scope"],
|
|
13
|
-
};
|
|
6
|
+
import { createStubHealthCheckCache } from "./cache-test-stub";
|
|
7
|
+
|
|
8
|
+
const passthroughCache = createStubHealthCheckCache();
|
|
14
9
|
|
|
15
10
|
describe("HealthCheck Router", () => {
|
|
16
11
|
const mockUser = {
|
package/src/router.ts
CHANGED
|
@@ -710,9 +710,10 @@ export const createHealthCheckRouter = (opts: {
|
|
|
710
710
|
),
|
|
711
711
|
getSystemHealthStatus: os.getSystemHealthStatus.handler(
|
|
712
712
|
async ({ input }) => {
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
713
|
+
// The cache holds the RAW (pre-override) status; incident overrides are
|
|
714
|
+
// folded downstream (always live) so an override lifts the instant its
|
|
715
|
+
// incident resolves. See ./cache.ts for the key/TTL/invalidation contract.
|
|
716
|
+
const base = await cache.read(input.systemId);
|
|
716
717
|
const folded = await foldIncidentOverrides({
|
|
717
718
|
[input.systemId]: base,
|
|
718
719
|
});
|
|
@@ -724,24 +725,15 @@ export const createHealthCheckRouter = (opts: {
|
|
|
724
725
|
async ({ input }) => {
|
|
725
726
|
// Per-entity caching: each system's status is cached individually
|
|
726
727
|
// and invalidated by id on mutations, so dashboards with overlapping
|
|
727
|
-
// (but non-identical) system sets share cache entries.
|
|
728
|
-
|
|
729
|
-
const statuses: Record<string, SystemHealthStatusResponse> = {};
|
|
730
|
-
await Promise.all(
|
|
731
|
-
input.systemIds.map(async (systemId) => {
|
|
732
|
-
statuses[systemId] = await cache.wrapSystemHealthStatus(
|
|
733
|
-
systemId,
|
|
734
|
-
() => service.getSystemHealthStatus(systemId),
|
|
735
|
-
);
|
|
736
|
-
}),
|
|
737
|
-
);
|
|
728
|
+
// (but non-identical) system sets share cache entries.
|
|
729
|
+
const statuses = await cache.readBulk(input.systemIds);
|
|
738
730
|
return { statuses: await foldIncidentOverrides(statuses) };
|
|
739
731
|
},
|
|
740
732
|
),
|
|
741
733
|
|
|
742
734
|
getBulkSystemHealthMatrix: os.getBulkSystemHealthMatrix.handler(
|
|
743
735
|
async ({ input }) => {
|
|
744
|
-
const matrix = await
|
|
736
|
+
const matrix = await cache.readMatrix(input.systemIds);
|
|
745
737
|
|
|
746
738
|
// Fold active incident overrides into each system's OVERALL rollup, so
|
|
747
739
|
// an incident-forced status still propagates through dependencies (as
|
package/src/schema.ts
CHANGED
|
@@ -127,6 +127,13 @@ export const systemHealthChecks = pgTable(
|
|
|
127
127
|
},
|
|
128
128
|
(t) => ({
|
|
129
129
|
pk: primaryKey({ columns: [t.systemId, t.configurationId] }),
|
|
130
|
+
// Reverse lookup for getSystemIdsForConfiguration on config-change
|
|
131
|
+
// recompute (WHERE configuration_id [AND enabled]). The PK leads with
|
|
132
|
+
// system_id, so a config-scoped scan cannot use it.
|
|
133
|
+
configEnabledIdx: index("system_health_checks_config_enabled_idx").on(
|
|
134
|
+
t.configurationId,
|
|
135
|
+
t.enabled,
|
|
136
|
+
),
|
|
130
137
|
}),
|
|
131
138
|
);
|
|
132
139
|
|
|
@@ -186,37 +193,65 @@ export const healthCheckStateTransitions = pgTable(
|
|
|
186
193
|
}),
|
|
187
194
|
);
|
|
188
195
|
|
|
189
|
-
export const healthCheckRuns = pgTable(
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
.
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
196
|
+
export const healthCheckRuns = pgTable(
|
|
197
|
+
"health_check_runs",
|
|
198
|
+
{
|
|
199
|
+
id: uuid("id").primaryKey().defaultRandom(),
|
|
200
|
+
configurationId: uuid("configuration_id")
|
|
201
|
+
.notNull()
|
|
202
|
+
.references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
|
|
203
|
+
systemId: text("system_id").notNull(),
|
|
204
|
+
/**
|
|
205
|
+
* Environment this run was executed for (per-environment fan-out).
|
|
206
|
+
* null = ran with no environment (the opt-out / no-membership case,
|
|
207
|
+
* which is exactly the pre-feature behavior). Nullable text, NOT a FK
|
|
208
|
+
* to the catalog `environments` table (healthcheck and catalog are
|
|
209
|
+
* separate plugins with separate Postgres schemas, mirroring how
|
|
210
|
+
* `systemId` is a bare text with no FK to `systems`).
|
|
211
|
+
*/
|
|
212
|
+
environmentId: text("environment_id"),
|
|
213
|
+
status: healthCheckStatusEnum("status").notNull(),
|
|
214
|
+
/** Execution duration in milliseconds */
|
|
215
|
+
latencyMs: integer("latency_ms"),
|
|
216
|
+
result: jsonb("result").$type<Record<string, unknown>>(),
|
|
217
|
+
/**
|
|
218
|
+
* Source identifier for result attribution.
|
|
219
|
+
* null = local core execution, UUID = satellite ID.
|
|
220
|
+
*/
|
|
221
|
+
sourceId: text("source_id"),
|
|
222
|
+
/**
|
|
223
|
+
* Human-readable source label for UI display.
|
|
224
|
+
* e.g. "Local" or "EU West (eu-west-1)".
|
|
225
|
+
*/
|
|
226
|
+
sourceLabel: text("source_label"),
|
|
227
|
+
timestamp: timestamp("timestamp").defaultNow().notNull(),
|
|
228
|
+
},
|
|
229
|
+
(t) => ({
|
|
230
|
+
// The status read path reads the last N runs for a (system, check) slice
|
|
231
|
+
// ordered by timestamp DESC. Without a supporting index every such read
|
|
232
|
+
// sequential-scans the whole (multi-million-row) table and sorts. Postgres
|
|
233
|
+
// scans a plain btree backward, so an ASC index serves ORDER BY ... DESC.
|
|
234
|
+
//
|
|
235
|
+
// Cross-environment newest-run read (WHERE system_id, configuration_id
|
|
236
|
+
// ORDER BY timestamp DESC LIMIT n - the single hottest query) and the
|
|
237
|
+
// retention DELETE (WHERE system_id, configuration_id AND timestamp < cutoff).
|
|
238
|
+
checkRecentIdx: index("health_check_runs_check_recent_idx").on(
|
|
239
|
+
t.systemId,
|
|
240
|
+
t.configurationId,
|
|
241
|
+
t.timestamp,
|
|
242
|
+
),
|
|
243
|
+
// Env-scoped slice read (WHERE system_id, configuration_id,
|
|
244
|
+
// environment_id [= ? | IS NULL] ORDER BY timestamp DESC LIMIT n), the
|
|
245
|
+
// per-check DISTINCT-environment discovery, and the per-env "last healthy"
|
|
246
|
+
// max(timestamp) GROUP BY environment_id.
|
|
247
|
+
sliceRecentIdx: index("health_check_runs_slice_recent_idx").on(
|
|
248
|
+
t.systemId,
|
|
249
|
+
t.configurationId,
|
|
250
|
+
t.environmentId,
|
|
251
|
+
t.timestamp,
|
|
252
|
+
),
|
|
253
|
+
}),
|
|
254
|
+
);
|
|
220
255
|
|
|
221
256
|
/**
|
|
222
257
|
* Bucket size enum for aggregated data.
|
|
@@ -286,5 +321,13 @@ export const healthCheckAggregates = pgTable(
|
|
|
286
321
|
t.bucketSize,
|
|
287
322
|
t.sourceId,
|
|
288
323
|
).nullsNotDistinct(),
|
|
324
|
+
// Health-state read with configuration_id absent (WHERE system_id,
|
|
325
|
+
// bucket_size, bucket_start ...). The unique index leads with
|
|
326
|
+
// configuration_id, so this query scans without a system-leading index.
|
|
327
|
+
systemBucketIdx: index("health_check_aggregates_system_bucket_idx").on(
|
|
328
|
+
t.systemId,
|
|
329
|
+
t.bucketSize,
|
|
330
|
+
t.bucketStart,
|
|
331
|
+
),
|
|
289
332
|
}),
|
|
290
333
|
);
|