@checkstack/healthcheck-backend 1.18.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +294 -0
- package/package.json +22 -20
- package/src/health-notification-content.test.ts +89 -0
- package/src/health-notification-content.ts +138 -0
- package/src/queue-executor.ts +31 -68
- package/src/router.ts +36 -0
- package/src/service-batching.test.ts +8 -0
- package/src/service-bulk-counts.it.test.ts +144 -0
- package/src/service-bulk-run-stats.it.test.ts +197 -0
- package/src/service-ordering.test.ts +6 -2
- package/src/service-paused-filter.test.ts +13 -0
- package/src/service-rollup-worst-wins.test.ts +209 -145
- package/src/service.ts +366 -185
- package/src/status-page/rollup.test.ts +40 -0
- package/src/status-page/rollup.ts +27 -0
- package/src/status-page/widgets.test.ts +303 -0
- package/src/status-page/widgets.ts +155 -39
package/src/service.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type HealthcheckSignalStatuses,
|
|
14
14
|
type RunStats,
|
|
15
15
|
stripEphemeralFields,
|
|
16
|
+
selectEffectiveEnvKeys,
|
|
16
17
|
} from "@checkstack/healthcheck-common";
|
|
17
18
|
import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
|
|
18
19
|
import { summarizeRuns, type StatRun } from "./run-stats.logic";
|
|
@@ -44,6 +45,7 @@ import {
|
|
|
44
45
|
isNotNull,
|
|
45
46
|
inArray,
|
|
46
47
|
max,
|
|
48
|
+
count,
|
|
47
49
|
} from "drizzle-orm";
|
|
48
50
|
import { ORPCError } from "@orpc/server";
|
|
49
51
|
import { evaluateHealthStatus } from "./state-evaluator";
|
|
@@ -987,6 +989,39 @@ export class HealthCheckService {
|
|
|
987
989
|
return results;
|
|
988
990
|
}
|
|
989
991
|
|
|
992
|
+
/**
|
|
993
|
+
* Count of health-check assignments for each of `systemIds`, keyed by
|
|
994
|
+
* systemId. ONE grouped query (no per-system fan-out) so the catalog manager
|
|
995
|
+
* can badge the whole visible list without an N+1 of {@link
|
|
996
|
+
* getSystemAssociations} - each of those held a pooled connection and
|
|
997
|
+
* contended with the run executor. Every requested system appears in the
|
|
998
|
+
* result (`0` when it has no `system_health_checks` rows, since the grouped
|
|
999
|
+
* query only returns systems that have at least one); the RPC's `recordKey`
|
|
1000
|
+
* gating then drops any keys the caller may not read.
|
|
1001
|
+
*/
|
|
1002
|
+
async getBulkAssignedHealthCheckCounts(
|
|
1003
|
+
systemIds: string[],
|
|
1004
|
+
): Promise<Record<string, number>> {
|
|
1005
|
+
// Seed 0 for every requested system so those with no assignments still
|
|
1006
|
+
// report a count (GROUP BY omits systems with zero rows).
|
|
1007
|
+
const counts: Record<string, number> = {};
|
|
1008
|
+
for (const id of systemIds) counts[id] = 0;
|
|
1009
|
+
|
|
1010
|
+
if (systemIds.length === 0) return counts;
|
|
1011
|
+
|
|
1012
|
+
const rows = await this.db
|
|
1013
|
+
.select({
|
|
1014
|
+
systemId: systemHealthChecks.systemId,
|
|
1015
|
+
assignmentCount: count(),
|
|
1016
|
+
})
|
|
1017
|
+
.from(systemHealthChecks)
|
|
1018
|
+
.where(inArray(systemHealthChecks.systemId, systemIds))
|
|
1019
|
+
.groupBy(systemHealthChecks.systemId);
|
|
1020
|
+
|
|
1021
|
+
for (const row of rows) counts[row.systemId] = row.assignmentCount;
|
|
1022
|
+
return counts;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
990
1025
|
/**
|
|
991
1026
|
* List the IDs of every system an ENABLED assignment of `configurationId`
|
|
992
1027
|
* targets. Used by the pause/resume RPC handlers to know which systems'
|
|
@@ -1097,6 +1132,12 @@ export class HealthCheckService {
|
|
|
1097
1132
|
stateThresholds: systemHealthChecks.stateThresholds,
|
|
1098
1133
|
configName: healthCheckConfigurations.name,
|
|
1099
1134
|
enabled: systemHealthChecks.enabled,
|
|
1135
|
+
// The per-assignment environment selector. Drives the rollup's
|
|
1136
|
+
// effective-slice filter below so a per-env slice whose environment
|
|
1137
|
+
// was DISABLED for this assignment (removed from `environmentIds`)
|
|
1138
|
+
// stops dragging the aggregate the instant it is disabled - instead
|
|
1139
|
+
// of lingering until its stale runs age out of the bounded window.
|
|
1140
|
+
environmentIds: systemHealthChecks.environmentIds,
|
|
1100
1141
|
})
|
|
1101
1142
|
.from(systemHealthChecks)
|
|
1102
1143
|
.innerJoin(
|
|
@@ -1167,44 +1208,71 @@ export class HealthCheckService {
|
|
|
1167
1208
|
// `rollup — worst-wins across environments within an association`).
|
|
1168
1209
|
// Per-env evaluation makes the rollup worst-wins stable regardless of
|
|
1169
1210
|
// insertion order or multi-pod racing.
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1211
|
+
//
|
|
1212
|
+
// Each env is windowed SEPARATELY (`maxWindowSize` runs PER env), not
|
|
1213
|
+
// via one shared `LIMIT maxWindowSize` across the mixed pool. A shared
|
|
1214
|
+
// window silently truncates a check that fans out to many envs: with
|
|
1215
|
+
// E envs each env sees only ~maxWindowSize/E of its own runs, so a
|
|
1216
|
+
// small consecutive threshold can miss a genuine per-env streak once
|
|
1217
|
+
// E grows. Per-env windows give every environment its full evaluation
|
|
1218
|
+
// depth regardless of how many siblings it has.
|
|
1219
|
+
const distinctEnvRows = await tx
|
|
1220
|
+
.selectDistinct({ environmentId: healthCheckRuns.environmentId })
|
|
1176
1221
|
.from(healthCheckRuns)
|
|
1177
1222
|
.where(
|
|
1178
1223
|
and(
|
|
1179
1224
|
eq(healthCheckRuns.systemId, systemId),
|
|
1180
1225
|
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1181
1226
|
),
|
|
1182
|
-
)
|
|
1183
|
-
|
|
1184
|
-
|
|
1227
|
+
);
|
|
1228
|
+
const presentEnvKeys = distinctEnvRows.map(
|
|
1229
|
+
(r) => r.environmentId ?? null,
|
|
1230
|
+
);
|
|
1185
1231
|
|
|
1186
|
-
//
|
|
1187
|
-
//
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1232
|
+
// Keep only slices that are still EFFECTIVE for this assignment: a
|
|
1233
|
+
// concrete environment removed from `environmentIds` (the reported
|
|
1234
|
+
// bug - "disable env for assignment"), plus the stale env-less slice
|
|
1235
|
+
// of a check that now fans out, are dropped. Without this a disabled
|
|
1236
|
+
// env's last unhealthy runs keep dragging the rollup via worst-wins,
|
|
1237
|
+
// because no health-change event fires for a slice that stopped
|
|
1238
|
+
// producing runs, so the event-driven rollup consumer never recomputes
|
|
1239
|
+
// it away. The selector is durable Postgres state (`environmentIds`),
|
|
1240
|
+
// so this is catalog-free and returns the same answer on every pod.
|
|
1241
|
+
const effectiveKeys = selectEffectiveEnvKeys({
|
|
1242
|
+
environmentIds: assoc.environmentIds,
|
|
1243
|
+
presentEnvKeys,
|
|
1244
|
+
});
|
|
1198
1245
|
|
|
1199
1246
|
status = "healthy";
|
|
1200
|
-
runsConsidered =
|
|
1201
|
-
lastRunAt =
|
|
1202
|
-
// Each env group is a slice. A check that has runs against N
|
|
1203
|
-
// currently fans out to N; before it has ever run
|
|
1204
|
-
//
|
|
1205
|
-
sliceCount = Math.max(
|
|
1247
|
+
runsConsidered = 0;
|
|
1248
|
+
lastRunAt = undefined;
|
|
1249
|
+
// Each EFFECTIVE env group is a slice. A check that has runs against N
|
|
1250
|
+
// effective envs currently fans out to N; before it has ever run (no
|
|
1251
|
+
// effective group) it is still one logical slice.
|
|
1252
|
+
sliceCount = Math.max(effectiveKeys.size, 1);
|
|
1206
1253
|
failingSliceCount = 0;
|
|
1207
|
-
for (const
|
|
1254
|
+
for (const key of effectiveKeys) {
|
|
1255
|
+
const envRuns = await tx
|
|
1256
|
+
.select({
|
|
1257
|
+
status: healthCheckRuns.status,
|
|
1258
|
+
timestamp: healthCheckRuns.timestamp,
|
|
1259
|
+
})
|
|
1260
|
+
.from(healthCheckRuns)
|
|
1261
|
+
.where(
|
|
1262
|
+
and(
|
|
1263
|
+
eq(healthCheckRuns.systemId, systemId),
|
|
1264
|
+
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1265
|
+
key === null
|
|
1266
|
+
? isNull(healthCheckRuns.environmentId)
|
|
1267
|
+
: eq(healthCheckRuns.environmentId, key),
|
|
1268
|
+
),
|
|
1269
|
+
)
|
|
1270
|
+
.orderBy(desc(healthCheckRuns.timestamp))
|
|
1271
|
+
.limit(maxWindowSize);
|
|
1272
|
+
|
|
1273
|
+
runsConsidered += envRuns.length;
|
|
1274
|
+
const newest = envRuns[0]?.timestamp;
|
|
1275
|
+
if (newest && (!lastRunAt || newest > lastRunAt)) lastRunAt = newest;
|
|
1208
1276
|
const envStatus = evaluateHealthStatus({ runs: envRuns, thresholds });
|
|
1209
1277
|
// Count EVERY failing slice (don't break early): the failing count
|
|
1210
1278
|
// feeds the dashboard numerator, so all non-healthy envs must tally.
|
|
@@ -1513,193 +1581,237 @@ export class HealthCheckService {
|
|
|
1513
1581
|
* Returns all health checks with their last 25 runs for sparkline visualization.
|
|
1514
1582
|
*/
|
|
1515
1583
|
async getSystemHealthOverview(systemId: string) {
|
|
1516
|
-
//
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
.from(systemHealthChecks)
|
|
1528
|
-
.innerJoin(
|
|
1529
|
-
healthCheckConfigurations,
|
|
1530
|
-
eq(systemHealthChecks.configurationId, healthCheckConfigurations.id),
|
|
1531
|
-
)
|
|
1532
|
-
.where(eq(systemHealthChecks.systemId, systemId));
|
|
1533
|
-
|
|
1534
|
-
const checks = [];
|
|
1535
|
-
const sparklineLimit = 25;
|
|
1536
|
-
|
|
1537
|
-
for (const assoc of associations) {
|
|
1538
|
-
// Get last 25 runs for sparkline (newest first, then reverse for chronological display)
|
|
1539
|
-
const runs = await this.db
|
|
1540
|
-
.select({
|
|
1541
|
-
id: healthCheckRuns.id,
|
|
1542
|
-
status: healthCheckRuns.status,
|
|
1543
|
-
timestamp: healthCheckRuns.timestamp,
|
|
1544
|
-
environmentId: healthCheckRuns.environmentId,
|
|
1545
|
-
})
|
|
1546
|
-
.from(healthCheckRuns)
|
|
1547
|
-
.where(
|
|
1548
|
-
and(
|
|
1549
|
-
eq(healthCheckRuns.systemId, systemId),
|
|
1550
|
-
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1551
|
-
),
|
|
1552
|
-
)
|
|
1553
|
-
.orderBy(desc(healthCheckRuns.timestamp))
|
|
1554
|
-
.limit(sparklineLimit);
|
|
1555
|
-
|
|
1556
|
-
// Reverse to chronological order (oldest first) for sparkline display
|
|
1557
|
-
const chronologicalRuns = runs.toReversed();
|
|
1558
|
-
|
|
1559
|
-
// Migrate and extract thresholds
|
|
1560
|
-
let thresholds: StateThresholds | undefined;
|
|
1561
|
-
if (assoc.stateThresholds) {
|
|
1562
|
-
thresholds = await stateThresholds.parse(assoc.stateThresholds);
|
|
1563
|
-
}
|
|
1564
|
-
|
|
1565
|
-
// Most recent HEALTHY run per environment, computed OUTSIDE the bounded
|
|
1566
|
-
// sparkline window so "last successful run" stays correct even when a
|
|
1567
|
-
// check has been failing for far longer than the last 25 runs. One
|
|
1568
|
-
// grouped aggregate query per check (env-less = the `null` group). The
|
|
1569
|
-
// (system_id, configuration_id, environment_id, timestamp) index makes
|
|
1570
|
-
// this a cheap max-per-group scan.
|
|
1571
|
-
const lastHealthyRows = await this.db
|
|
1584
|
+
// §perf: batch the 1 (associations) + N·(2+E) (per-check recentRuns +
|
|
1585
|
+
// grouped last-healthy + per-env slice) reads into ONE scoped transaction
|
|
1586
|
+
// so the whole read fan-out pays a single BEGIN/SET LOCAL/COMMIT and holds
|
|
1587
|
+
// one connection, instead of 1+N·(2+E) standalone scoped queries each
|
|
1588
|
+
// checking a connection out. Only pure CPU (stateThresholds.parse,
|
|
1589
|
+
// evaluateHealthStatus, selectEffectiveEnvKeys) sits between the queries —
|
|
1590
|
+
// no DB-external await — so wrapping is safe. This mirrors the sibling
|
|
1591
|
+
// getSystemHealthStatus above. See withScopedTransaction.
|
|
1592
|
+
const checks = await withScopedTransaction(this.db, async (tx) => {
|
|
1593
|
+
// Get all associations with config details
|
|
1594
|
+
const associations = await tx
|
|
1572
1595
|
.select({
|
|
1573
|
-
|
|
1574
|
-
|
|
1596
|
+
configurationId: systemHealthChecks.configurationId,
|
|
1597
|
+
configName: healthCheckConfigurations.name,
|
|
1598
|
+
strategyId: healthCheckConfigurations.strategyId,
|
|
1599
|
+
intervalSeconds: healthCheckConfigurations.intervalSeconds,
|
|
1600
|
+
enabled: systemHealthChecks.enabled,
|
|
1601
|
+
paused: healthCheckConfigurations.paused,
|
|
1602
|
+
stateThresholds: systemHealthChecks.stateThresholds,
|
|
1603
|
+
// The per-assignment environment selector, surfaced so the check-level
|
|
1604
|
+
// rollup status here excludes slices whose env was disabled for this
|
|
1605
|
+
// assignment, and so the response can carry it to the frontend orphan
|
|
1606
|
+
// detection (a disabled-for-assignment env is tucked under "Old checks"
|
|
1607
|
+
// even though it is still part of the system's membership).
|
|
1608
|
+
environmentIds: systemHealthChecks.environmentIds,
|
|
1575
1609
|
})
|
|
1576
|
-
.from(
|
|
1577
|
-
.
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1581
|
-
eq(healthCheckRuns.status, "healthy"),
|
|
1582
|
-
),
|
|
1610
|
+
.from(systemHealthChecks)
|
|
1611
|
+
.innerJoin(
|
|
1612
|
+
healthCheckConfigurations,
|
|
1613
|
+
eq(systemHealthChecks.configurationId, healthCheckConfigurations.id),
|
|
1583
1614
|
)
|
|
1584
|
-
.
|
|
1585
|
-
const lastHealthyByEnv = new Map<string | null, Date>();
|
|
1586
|
-
let checkLastSuccessfulRunAt: Date | undefined;
|
|
1587
|
-
for (const row of lastHealthyRows) {
|
|
1588
|
-
if (!row.lastSuccessAt) continue;
|
|
1589
|
-
lastHealthyByEnv.set(row.environmentId ?? null, row.lastSuccessAt);
|
|
1590
|
-
if (
|
|
1591
|
-
!checkLastSuccessfulRunAt ||
|
|
1592
|
-
row.lastSuccessAt > checkLastSuccessfulRunAt
|
|
1593
|
-
) {
|
|
1594
|
-
checkLastSuccessfulRunAt = row.lastSuccessAt;
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1615
|
+
.where(eq(systemHealthChecks.systemId, systemId));
|
|
1597
1616
|
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
// monotonic run window and worst-wins across envs — this is the same
|
|
1601
|
-
// derivation `getSystemHealthStatus(systemId)` uses for the rollup; see
|
|
1602
|
-
// that method for the rationale (flattening envs feeds interleaved
|
|
1603
|
-
// statuses to the consecutive evaluator and masks sibling outages).
|
|
1604
|
-
const perEnvironment: {
|
|
1605
|
-
environmentId: string | null;
|
|
1606
|
-
status: HealthCheckStatus;
|
|
1607
|
-
lastSuccessfulRunAt?: Date;
|
|
1608
|
-
recentRuns: { id: string; status: HealthCheckStatus; timestamp: Date }[];
|
|
1609
|
-
}[] = [];
|
|
1610
|
-
|
|
1611
|
-
// Stable ordering of env keys: env-less (`null`) first, then env ids in
|
|
1612
|
-
// the order they were first encountered in the mixed pool (membership
|
|
1613
|
-
// order is otherwise unobservable here without a catalog read; recent
|
|
1614
|
-
// runs surface stable, recent order).
|
|
1615
|
-
const envKeys: (string | null)[] = [];
|
|
1616
|
-
const seenEnv = new Set<string | null>();
|
|
1617
|
-
for (const r of runs) {
|
|
1618
|
-
const key = r.environmentId ?? null;
|
|
1619
|
-
if (!seenEnv.has(key)) {
|
|
1620
|
-
seenEnv.add(key);
|
|
1621
|
-
envKeys.push(key);
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
// If no runs at all, surface a single env-less entry so UI can render
|
|
1625
|
-
// an empty row rather than nothing.
|
|
1626
|
-
if (envKeys.length === 0) envKeys.push(null);
|
|
1617
|
+
const checks = [];
|
|
1618
|
+
const sparklineLimit = 25;
|
|
1627
1619
|
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
const
|
|
1620
|
+
for (const assoc of associations) {
|
|
1621
|
+
// Get last 25 runs for sparkline (newest first, then reverse for chronological display)
|
|
1622
|
+
const runs = await tx
|
|
1631
1623
|
.select({
|
|
1632
1624
|
id: healthCheckRuns.id,
|
|
1633
1625
|
status: healthCheckRuns.status,
|
|
1634
1626
|
timestamp: healthCheckRuns.timestamp,
|
|
1627
|
+
environmentId: healthCheckRuns.environmentId,
|
|
1635
1628
|
})
|
|
1636
1629
|
.from(healthCheckRuns)
|
|
1637
1630
|
.where(
|
|
1638
1631
|
and(
|
|
1639
1632
|
eq(healthCheckRuns.systemId, systemId),
|
|
1640
1633
|
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1641
|
-
envId === null
|
|
1642
|
-
? isNull(healthCheckRuns.environmentId)
|
|
1643
|
-
: eq(healthCheckRuns.environmentId, envId),
|
|
1644
1634
|
),
|
|
1645
1635
|
)
|
|
1646
1636
|
.orderBy(desc(healthCheckRuns.timestamp))
|
|
1647
1637
|
.limit(sparklineLimit);
|
|
1648
1638
|
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1639
|
+
// Reverse to chronological order (oldest first) for sparkline display
|
|
1640
|
+
const chronologicalRuns = runs.toReversed();
|
|
1641
|
+
|
|
1642
|
+
// Migrate and extract thresholds
|
|
1643
|
+
let thresholds: StateThresholds | undefined;
|
|
1644
|
+
if (assoc.stateThresholds) {
|
|
1645
|
+
thresholds = await stateThresholds.parse(assoc.stateThresholds);
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
// Most recent HEALTHY run per environment, computed OUTSIDE the bounded
|
|
1649
|
+
// sparkline window so "last successful run" stays correct even when a
|
|
1650
|
+
// check has been failing for far longer than the last 25 runs. One
|
|
1651
|
+
// grouped aggregate query per check (env-less = the `null` group). The
|
|
1652
|
+
// (system_id, configuration_id, environment_id, timestamp) index makes
|
|
1653
|
+
// this a cheap max-per-group scan.
|
|
1654
|
+
const lastHealthyRows = await tx
|
|
1655
|
+
.select({
|
|
1656
|
+
environmentId: healthCheckRuns.environmentId,
|
|
1657
|
+
lastSuccessAt: max(healthCheckRuns.timestamp),
|
|
1658
|
+
})
|
|
1659
|
+
.from(healthCheckRuns)
|
|
1660
|
+
.where(
|
|
1661
|
+
and(
|
|
1662
|
+
eq(healthCheckRuns.systemId, systemId),
|
|
1663
|
+
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1664
|
+
eq(healthCheckRuns.status, "healthy"),
|
|
1665
|
+
),
|
|
1666
|
+
)
|
|
1667
|
+
.groupBy(healthCheckRuns.environmentId);
|
|
1668
|
+
const lastHealthyByEnv = new Map<string | null, Date>();
|
|
1669
|
+
let checkLastSuccessfulRunAt: Date | undefined;
|
|
1670
|
+
for (const row of lastHealthyRows) {
|
|
1671
|
+
if (!row.lastSuccessAt) continue;
|
|
1672
|
+
lastHealthyByEnv.set(row.environmentId ?? null, row.lastSuccessAt);
|
|
1673
|
+
if (
|
|
1674
|
+
!checkLastSuccessfulRunAt ||
|
|
1675
|
+
row.lastSuccessAt > checkLastSuccessfulRunAt
|
|
1676
|
+
) {
|
|
1677
|
+
checkLastSuccessfulRunAt = row.lastSuccessAt;
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
// Group the fetched runs by environmentId (null = env-less slice). We
|
|
1682
|
+
// query each env's slice separately below to evaluate it on its own
|
|
1683
|
+
// monotonic run window and worst-wins across envs — this is the same
|
|
1684
|
+
// derivation `getSystemHealthStatus(systemId)` uses for the rollup; see
|
|
1685
|
+
// that method for the rationale (flattening envs feeds interleaved
|
|
1686
|
+
// statuses to the consecutive evaluator and masks sibling outages).
|
|
1687
|
+
const perEnvironment: {
|
|
1688
|
+
environmentId: string | null;
|
|
1689
|
+
status: HealthCheckStatus;
|
|
1690
|
+
lastSuccessfulRunAt?: Date;
|
|
1691
|
+
recentRuns: {
|
|
1692
|
+
id: string;
|
|
1693
|
+
status: HealthCheckStatus;
|
|
1694
|
+
timestamp: Date;
|
|
1695
|
+
}[];
|
|
1696
|
+
}[] = [];
|
|
1697
|
+
|
|
1698
|
+
// Stable ordering of env keys: env-less (`null`) first, then env ids in
|
|
1699
|
+
// the order they were first encountered in the mixed pool (membership
|
|
1700
|
+
// order is otherwise unobservable here without a catalog read; recent
|
|
1701
|
+
// runs surface stable, recent order).
|
|
1702
|
+
const envKeys: (string | null)[] = [];
|
|
1703
|
+
const seenEnv = new Set<string | null>();
|
|
1704
|
+
for (const r of runs) {
|
|
1705
|
+
const key = r.environmentId ?? null;
|
|
1706
|
+
if (!seenEnv.has(key)) {
|
|
1707
|
+
seenEnv.add(key);
|
|
1708
|
+
envKeys.push(key);
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
// If no runs at all, surface a single env-less entry so UI can render
|
|
1712
|
+
// an empty row rather than nothing.
|
|
1713
|
+
if (envKeys.length === 0) envKeys.push(null);
|
|
1714
|
+
|
|
1715
|
+
// The slices that currently CONTRIBUTE to this check's rollup status: a
|
|
1716
|
+
// concrete env removed from `environmentIds` (disabled for the assignment)
|
|
1717
|
+
// and the stale env-less slice of a check that now fans out are excluded,
|
|
1718
|
+
// mirroring `getSystemHealthStatus`. The orphaned slices are still emitted
|
|
1719
|
+
// in `perEnvironment` (the frontend tucks them under "Old checks"); they
|
|
1720
|
+
// just no longer drag the check-level worst-wins `status`.
|
|
1721
|
+
const effectiveKeys = selectEffectiveEnvKeys({
|
|
1722
|
+
environmentIds: assoc.environmentIds,
|
|
1723
|
+
presentEnvKeys: envKeys,
|
|
1652
1724
|
});
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1725
|
+
|
|
1726
|
+
let aggregateStatus: HealthCheckStatus = "healthy";
|
|
1727
|
+
for (const envId of envKeys) {
|
|
1728
|
+
const envRuns = await tx
|
|
1729
|
+
.select({
|
|
1730
|
+
id: healthCheckRuns.id,
|
|
1731
|
+
status: healthCheckRuns.status,
|
|
1732
|
+
timestamp: healthCheckRuns.timestamp,
|
|
1733
|
+
})
|
|
1734
|
+
.from(healthCheckRuns)
|
|
1735
|
+
.where(
|
|
1736
|
+
and(
|
|
1737
|
+
eq(healthCheckRuns.systemId, systemId),
|
|
1738
|
+
eq(healthCheckRuns.configurationId, assoc.configurationId),
|
|
1739
|
+
envId === null
|
|
1740
|
+
? isNull(healthCheckRuns.environmentId)
|
|
1741
|
+
: eq(healthCheckRuns.environmentId, envId),
|
|
1742
|
+
),
|
|
1743
|
+
)
|
|
1744
|
+
.orderBy(desc(healthCheckRuns.timestamp))
|
|
1745
|
+
.limit(sparklineLimit);
|
|
1746
|
+
|
|
1747
|
+
const envStatus = evaluateHealthStatus({
|
|
1748
|
+
runs: envRuns,
|
|
1749
|
+
thresholds,
|
|
1750
|
+
});
|
|
1751
|
+
// Worst-wins across EFFECTIVE envs (unhealthy > degraded > healthy).
|
|
1752
|
+
// An orphaned slice's status is still reported per-env but must not
|
|
1753
|
+
// move the aggregate.
|
|
1754
|
+
if (effectiveKeys.has(envId)) {
|
|
1755
|
+
if (envStatus === "unhealthy") {
|
|
1756
|
+
aggregateStatus = "unhealthy";
|
|
1757
|
+
} else if (
|
|
1758
|
+
envStatus === "degraded" &&
|
|
1759
|
+
aggregateStatus === "healthy"
|
|
1760
|
+
) {
|
|
1761
|
+
aggregateStatus = "degraded";
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
perEnvironment.push({
|
|
1766
|
+
environmentId: envId,
|
|
1767
|
+
status: envStatus,
|
|
1768
|
+
lastSuccessfulRunAt: lastHealthyByEnv.get(envId),
|
|
1769
|
+
recentRuns: envRuns.toReversed().map((r) => ({
|
|
1770
|
+
id: r.id,
|
|
1771
|
+
status: r.status,
|
|
1772
|
+
timestamp: r.timestamp,
|
|
1773
|
+
})),
|
|
1774
|
+
});
|
|
1658
1775
|
}
|
|
1659
1776
|
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1777
|
+
// Evaluate current status (runs are in DESC order - newest first - as evaluateHealthStatus expects).
|
|
1778
|
+
// For a paused configuration the runs are stale (execution is skipped),
|
|
1779
|
+
// so the evaluated `status` is NOT a meaningful current verdict — the
|
|
1780
|
+
// frontend renders a "Paused" pill from the `paused` flag instead.
|
|
1781
|
+
// We still compute it so the historical/sparkline path stays uniform,
|
|
1782
|
+
// and so a non-paused consumer that ignores `paused` sees a best-
|
|
1783
|
+
// effort status rather than a hard null. `aggregateStatus` is the
|
|
1784
|
+
// worst-wins-across-envs rollup derived above (it equals what
|
|
1785
|
+
// evaluateHealthStatus would return on the flat pool if only ONE env is
|
|
1786
|
+
// present, preserving per-check single-env behavior).
|
|
1787
|
+
const status = aggregateStatus;
|
|
1788
|
+
|
|
1789
|
+
checks.push({
|
|
1790
|
+
configurationId: assoc.configurationId,
|
|
1791
|
+
configurationName: assoc.configName,
|
|
1792
|
+
strategyId: assoc.strategyId,
|
|
1793
|
+
intervalSeconds: assoc.intervalSeconds,
|
|
1794
|
+
enabled: assoc.enabled,
|
|
1795
|
+
paused: assoc.paused,
|
|
1796
|
+
status,
|
|
1797
|
+
stateThresholds: thresholds,
|
|
1798
|
+
// Surface the per-assignment environment selector so the frontend can
|
|
1799
|
+
// treat a slice whose env was disabled for THIS assignment as orphaned
|
|
1800
|
+
// (system membership alone can't tell - the env is still in the system).
|
|
1801
|
+
environmentIds: assoc.environmentIds,
|
|
1802
|
+
lastSuccessfulRunAt: checkLastSuccessfulRunAt,
|
|
1803
|
+
recentRuns: chronologicalRuns.map((r) => ({
|
|
1665
1804
|
id: r.id,
|
|
1666
1805
|
status: r.status,
|
|
1667
1806
|
timestamp: r.timestamp,
|
|
1807
|
+
environmentId: r.environmentId,
|
|
1668
1808
|
})),
|
|
1809
|
+
perEnvironment,
|
|
1669
1810
|
});
|
|
1670
1811
|
}
|
|
1671
1812
|
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
// so the evaluated `status` is NOT a meaningful current verdict — the
|
|
1675
|
-
// frontend renders a "Paused" pill from the `paused` flag instead.
|
|
1676
|
-
// We still compute it so the historical/sparkline path stays uniform,
|
|
1677
|
-
// and so a non-paused consumer that ignores `paused` sees a best-
|
|
1678
|
-
// effort status rather than a hard null. `aggregateStatus` is the
|
|
1679
|
-
// worst-wins-across-envs rollup derived above (it equals what
|
|
1680
|
-
// evaluateHealthStatus would return on the flat pool if only ONE env is
|
|
1681
|
-
// present, preserving per-check single-env behavior).
|
|
1682
|
-
const status = aggregateStatus;
|
|
1683
|
-
|
|
1684
|
-
checks.push({
|
|
1685
|
-
configurationId: assoc.configurationId,
|
|
1686
|
-
configurationName: assoc.configName,
|
|
1687
|
-
strategyId: assoc.strategyId,
|
|
1688
|
-
intervalSeconds: assoc.intervalSeconds,
|
|
1689
|
-
enabled: assoc.enabled,
|
|
1690
|
-
paused: assoc.paused,
|
|
1691
|
-
status,
|
|
1692
|
-
stateThresholds: thresholds,
|
|
1693
|
-
lastSuccessfulRunAt: checkLastSuccessfulRunAt,
|
|
1694
|
-
recentRuns: chronologicalRuns.map((r) => ({
|
|
1695
|
-
id: r.id,
|
|
1696
|
-
status: r.status,
|
|
1697
|
-
timestamp: r.timestamp,
|
|
1698
|
-
environmentId: r.environmentId,
|
|
1699
|
-
})),
|
|
1700
|
-
perEnvironment,
|
|
1701
|
-
});
|
|
1702
|
-
}
|
|
1813
|
+
return checks;
|
|
1814
|
+
});
|
|
1703
1815
|
|
|
1704
1816
|
return { systemId, checks };
|
|
1705
1817
|
}
|
|
@@ -1814,6 +1926,7 @@ export class HealthCheckService {
|
|
|
1814
1926
|
sourceFilter?: string;
|
|
1815
1927
|
statusFilter?: HealthCheckStatus[];
|
|
1816
1928
|
environmentId?: string | null;
|
|
1929
|
+
environmentIds?: string[];
|
|
1817
1930
|
maxBuckets?: number;
|
|
1818
1931
|
}): Promise<RunStats> {
|
|
1819
1932
|
const {
|
|
@@ -1824,6 +1937,7 @@ export class HealthCheckService {
|
|
|
1824
1937
|
sourceFilter,
|
|
1825
1938
|
statusFilter,
|
|
1826
1939
|
environmentId,
|
|
1940
|
+
environmentIds,
|
|
1827
1941
|
maxBuckets = 24,
|
|
1828
1942
|
} = props;
|
|
1829
1943
|
|
|
@@ -1848,6 +1962,12 @@ export class HealthCheckService {
|
|
|
1848
1962
|
} else if (environmentId !== undefined) {
|
|
1849
1963
|
conditions.push(eq(healthCheckRuns.environmentId, environmentId));
|
|
1850
1964
|
}
|
|
1965
|
+
// Set-of-environments filter (the status page's per-page env scope). Only
|
|
1966
|
+
// counts runs tagged with one of the selected envs; env-less runs are
|
|
1967
|
+
// excluded (they belong to no published environment).
|
|
1968
|
+
if (environmentIds && environmentIds.length > 0) {
|
|
1969
|
+
conditions.push(inArray(healthCheckRuns.environmentId, environmentIds));
|
|
1970
|
+
}
|
|
1851
1971
|
|
|
1852
1972
|
const rows = await this.db
|
|
1853
1973
|
.select({
|
|
@@ -1867,6 +1987,67 @@ export class HealthCheckService {
|
|
|
1867
1987
|
return summarizeRuns({ runs, startDate, endDate, maxBuckets });
|
|
1868
1988
|
}
|
|
1869
1989
|
|
|
1990
|
+
/**
|
|
1991
|
+
* Bulk variant of {@link getRunStats}: compute the compact stats summary for
|
|
1992
|
+
* MANY systems over one shared window in a SINGLE grouped read, instead of an
|
|
1993
|
+
* N+1 fan-out of per-system `getRunStats` calls. Fetches every matching run
|
|
1994
|
+
* for all `systemIds` with one `inArray` query, groups by systemId in JS, and
|
|
1995
|
+
* runs the same pure `summarizeRuns` per system so each entry is identical to
|
|
1996
|
+
* what `getRunStats({ systemId })` would return. Systems with no runs in the
|
|
1997
|
+
* window are OMITTED from the record (they simply have no rows to group),
|
|
1998
|
+
* matching the single endpoint's "no runs => nothing to report" semantics.
|
|
1999
|
+
*/
|
|
2000
|
+
async getBulkRunStats(props: {
|
|
2001
|
+
systemIds: string[];
|
|
2002
|
+
startDate: Date;
|
|
2003
|
+
endDate: Date;
|
|
2004
|
+
environmentIds?: string[];
|
|
2005
|
+
maxBuckets?: number;
|
|
2006
|
+
}): Promise<Record<string, RunStats>> {
|
|
2007
|
+
const { systemIds, startDate, endDate, environmentIds, maxBuckets = 24 } =
|
|
2008
|
+
props;
|
|
2009
|
+
if (systemIds.length === 0) return {};
|
|
2010
|
+
|
|
2011
|
+
const conditions = [
|
|
2012
|
+
inArray(healthCheckRuns.systemId, systemIds),
|
|
2013
|
+
gte(healthCheckRuns.timestamp, startDate),
|
|
2014
|
+
lte(healthCheckRuns.timestamp, endDate),
|
|
2015
|
+
];
|
|
2016
|
+
// Set-of-environments filter (the status page's per-page env scope): count
|
|
2017
|
+
// only runs tagged with one of the selected envs; env-less runs excluded.
|
|
2018
|
+
if (environmentIds && environmentIds.length > 0) {
|
|
2019
|
+
conditions.push(inArray(healthCheckRuns.environmentId, environmentIds));
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
const rows = await this.db
|
|
2023
|
+
.select({
|
|
2024
|
+
systemId: healthCheckRuns.systemId,
|
|
2025
|
+
timestamp: healthCheckRuns.timestamp,
|
|
2026
|
+
status: healthCheckRuns.status,
|
|
2027
|
+
latencyMs: healthCheckRuns.latencyMs,
|
|
2028
|
+
})
|
|
2029
|
+
.from(healthCheckRuns)
|
|
2030
|
+
.where(and(...conditions));
|
|
2031
|
+
|
|
2032
|
+
const bySystem = new Map<string, StatRun[]>();
|
|
2033
|
+
for (const r of rows) {
|
|
2034
|
+
const list = bySystem.get(r.systemId);
|
|
2035
|
+
const run: StatRun = {
|
|
2036
|
+
timestamp: r.timestamp,
|
|
2037
|
+
status: r.status,
|
|
2038
|
+
latencyMs: r.latencyMs ?? undefined,
|
|
2039
|
+
};
|
|
2040
|
+
if (list) list.push(run);
|
|
2041
|
+
else bySystem.set(r.systemId, [run]);
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
const out: Record<string, RunStats> = {};
|
|
2045
|
+
for (const [systemId, runs] of bySystem) {
|
|
2046
|
+
out[systemId] = summarizeRuns({ runs, startDate, endDate, maxBuckets });
|
|
2047
|
+
}
|
|
2048
|
+
return out;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
1870
2051
|
/**
|
|
1871
2052
|
* Get detailed health check run history with full result data.
|
|
1872
2053
|
* Restricted to users with manage access.
|