@checkstack/healthcheck-backend 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +211 -0
- package/package.json +29 -29
- package/src/aggregation-utils.test.ts +132 -0
- package/src/aggregation-utils.ts +70 -6
- package/src/collector-assertions.test.ts +97 -1
- package/src/collector-assertions.ts +141 -34
- package/src/index.ts +29 -0
- package/src/queue-executor.test.ts +182 -0
- package/src/queue-executor.ts +12 -3
- package/src/realtime-aggregation.test.ts +136 -0
- package/src/realtime-aggregation.ts +39 -2
- package/src/retention-job.ts +64 -1
- package/src/retention-rollup.test.ts +69 -0
- package/src/router.ts +92 -5
- package/src/service-ingest-assertions.test.ts +213 -0
- package/src/service.ts +179 -9
- package/src/status-page/widgets.ts +11 -1
- package/src/system-health-override.test.ts +94 -0
- package/src/system-health-override.ts +93 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,216 @@
|
|
|
1
1
|
# @checkstack/healthcheck-backend
|
|
2
2
|
|
|
3
|
+
## 1.17.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 390d9cf: Add a **Container** health-check strategy for monitoring Docker and Podman
|
|
8
|
+
containers that expose no external service of their own. It reports container
|
|
9
|
+
existence, running state, healthcheck status, exit code, restart count, and
|
|
10
|
+
OOM-killed via the **Container Status** collector, and CPU/memory usage via the
|
|
11
|
+
**Container Stats** collector. Both collectors issue only read (GET) requests
|
|
12
|
+
against the runtime REST API.
|
|
13
|
+
|
|
14
|
+
The check runs wherever the executor runs: locally on the core instance (the
|
|
15
|
+
default) to watch containers that share a host with Checkstack, or on a
|
|
16
|
+
satellite pinned to another host.
|
|
17
|
+
|
|
18
|
+
Critically, Checkstack never touches the raw container socket. The strategy
|
|
19
|
+
talks the Docker Engine / Podman libpod API over either a unix socket path or an
|
|
20
|
+
`http(s)` endpoint, so operators point it at a **read-only socket-proxy**
|
|
21
|
+
(`lscr.io/linuxserver/socket-proxy` with `POST=0`) running next to whichever
|
|
22
|
+
Checkstack instance runs the check - core or a satellite - or at a rootless
|
|
23
|
+
Podman socket. The raw socket is mounted only into the proxy; even a compromised
|
|
24
|
+
instance can only read container state, never control the host. A stopped or missing container is a successful collection whose metrics
|
|
25
|
+
feed assertions (following the transport-failure-vs-metric rule) - only an
|
|
26
|
+
unreachable runtime endpoint fails the check. Container `exec` probes are
|
|
27
|
+
intentionally not offered because they would require write access to the socket.
|
|
28
|
+
|
|
29
|
+
To support in-product setup guidance, the health-check strategy contract gains
|
|
30
|
+
an optional `setupInstructions` (Markdown) field, surfaced in the DTO and
|
|
31
|
+
rendered as a collapsible "Setup guide" callout above the strategy config fields
|
|
32
|
+
in the editor. The Container strategy populates it with the secure proxy setup.
|
|
33
|
+
|
|
34
|
+
The hardened socket-proxy compose is maintained as a single canonical file
|
|
35
|
+
(`deploy/socket-proxy/docker-compose.yml`) that operators `include:` from their
|
|
36
|
+
core or satellite compose, so the read-only / `POST=0` / internal-network
|
|
37
|
+
hardening is defined in exactly one place; the docs and the in-product setup
|
|
38
|
+
guide reference it rather than duplicating the YAML.
|
|
39
|
+
|
|
40
|
+
Also removes a stale hand-written `HealthCheckStrategyDto` interface in
|
|
41
|
+
`@checkstack/healthcheck-common` that shadowed (and lagged behind) the
|
|
42
|
+
Zod-inferred DTO; the inferred type from `schemas.ts` is now the single source
|
|
43
|
+
of truth and correctly carries `resultSchema`, `aggregatedResultSchema`, and the
|
|
44
|
+
new `setupInstructions`.
|
|
45
|
+
|
|
46
|
+
Thanks to [@stuajnht](https://github.com/stuajnht) for the valuable feedback
|
|
47
|
+
that shaped this release.
|
|
48
|
+
|
|
49
|
+
- fc64fad: Dependencies can now be scoped to a specific environment and/or health check of
|
|
50
|
+
the upstream system, each with its own severity - a "matrix" of scope cells.
|
|
51
|
+
|
|
52
|
+
Previously a dependency watched the upstream's overall health (any check, any
|
|
53
|
+
environment) at the edge's impact type, with optional per-check rules. That
|
|
54
|
+
default is unchanged: with no scope cells configured, the dependency behaves
|
|
55
|
+
exactly as before. Now each cell pins a check (a specific configuration, or
|
|
56
|
+
"any"), an environment (a specific environment, or "any"), and a severity
|
|
57
|
+
(informational / degraded / critical). When a dependency has any cells, only
|
|
58
|
+
those slices are watched (they replace the whole-system watch) and the worst
|
|
59
|
+
result across cells wins. This lets you express, e.g., "System A depends on
|
|
60
|
+
System B only in `prod`", or "only when B's TLS check in `prod` fails", and lets
|
|
61
|
+
different cells carry different severities.
|
|
62
|
+
|
|
63
|
+
Because each environment is evaluated on its own slice, a scoped dependency
|
|
64
|
+
catches an environment-specific outage that the upstream's overall status
|
|
65
|
+
(worst-wins across environments) would otherwise hide. The dependency evaluator
|
|
66
|
+
now reads per-(check, environment) health via a new
|
|
67
|
+
`@checkstack/healthcheck-common` bulk contract `getBulkSystemHealthMatrix` (and
|
|
68
|
+
its `@checkstack/healthcheck-backend` implementation), which returns each
|
|
69
|
+
system's cross-environment rollup plus a per-environment slice. Incident
|
|
70
|
+
overrides still fold into the overall rollup, so incident-forced statuses keep
|
|
71
|
+
propagating through dependencies.
|
|
72
|
+
|
|
73
|
+
The scope-cell store gains a nullable `environment_id` column and makes
|
|
74
|
+
`health_check_id` nullable (forward-only migration; existing rows keep working
|
|
75
|
+
as "any check, any environment"). The dependency editor's per-check panel
|
|
76
|
+
becomes a scope-matrix editor with check + environment + severity rows.
|
|
77
|
+
|
|
78
|
+
Transitive (multi-hop) dependencies still cascade using the upstream's overall
|
|
79
|
+
status; per-environment cascades across multiple hops are not yet propagated.
|
|
80
|
+
|
|
81
|
+
- 9d30324: Incidents can now optionally override the health status of their affected
|
|
82
|
+
systems. When creating or editing an incident you can pick "Override system
|
|
83
|
+
health" (Degraded or Unhealthy); while the incident is active (not resolved)
|
|
84
|
+
that status is folded into every affected system's derived health via
|
|
85
|
+
worst-wins, so it shows on every health surface (status pages, dashboards,
|
|
86
|
+
dependency map, catalog badges). A health check reporting a worse status still
|
|
87
|
+
wins, and the override lifts automatically when the incident resolves. This
|
|
88
|
+
covers components that no automated check can monitor (e.g. a running app whose
|
|
89
|
+
licenses were revoked so it won't open).
|
|
90
|
+
|
|
91
|
+
The override is a deliberate operator choice, independent of the incident's
|
|
92
|
+
severity. A new service-typed incident RPC `getActiveHealthOverrides` exposes
|
|
93
|
+
active overrides per system, which `@checkstack/healthcheck-backend` reads and
|
|
94
|
+
folds into `getSystemHealthStatus`. The system-health response gains an optional
|
|
95
|
+
`override` field naming the contributing incident so UIs can explain why a
|
|
96
|
+
system reads unhealthy when its checks look fine. The system health badge uses
|
|
97
|
+
it to show, on hover, when a status was forced by an incident.
|
|
98
|
+
|
|
99
|
+
The dashboard "problem system" signal attributes an override-forced status to
|
|
100
|
+
the incident ("Forced by incident: <title>") instead of misreporting
|
|
101
|
+
"0 of N checks failing", while a genuinely worse health check still drives the
|
|
102
|
+
signal and its detail. Public status pages reflect the forced status but never
|
|
103
|
+
carry the incident title (the widget DTOs project only the status), so an
|
|
104
|
+
override cannot leak the name of a hidden incident.
|
|
105
|
+
|
|
106
|
+
Behavior change: a system's derived health now reflects active incident
|
|
107
|
+
overrides in addition to its health checks. Adds a forward-only migration for
|
|
108
|
+
the new nullable `incidents.health_override` column.
|
|
109
|
+
|
|
110
|
+
Thanks to [@stuajnht](https://github.com/stuajnht) for the valuable feedback
|
|
111
|
+
that shaped this release.
|
|
112
|
+
|
|
113
|
+
### Patch Changes
|
|
114
|
+
|
|
115
|
+
- Updated dependencies [390d9cf]
|
|
116
|
+
- Updated dependencies [390d9cf]
|
|
117
|
+
- Updated dependencies [fc64fad]
|
|
118
|
+
- Updated dependencies [fc64fad]
|
|
119
|
+
- Updated dependencies [9d30324]
|
|
120
|
+
- Updated dependencies [9d30324]
|
|
121
|
+
- Updated dependencies [b218e3e]
|
|
122
|
+
- @checkstack/ai-backend@0.10.8
|
|
123
|
+
- @checkstack/backend-api@0.30.0
|
|
124
|
+
- @checkstack/healthcheck-common@1.14.0
|
|
125
|
+
- @checkstack/incident-common@1.8.0
|
|
126
|
+
- @checkstack/incident-backend@1.10.0
|
|
127
|
+
- @checkstack/automation-backend@0.10.10
|
|
128
|
+
- @checkstack/catalog-backend@1.6.8
|
|
129
|
+
- @checkstack/command-backend@0.2.20
|
|
130
|
+
- @checkstack/gitops-backend@0.5.20
|
|
131
|
+
- @checkstack/satellite-backend@0.8.2
|
|
132
|
+
- @checkstack/script-packages-backend@0.3.24
|
|
133
|
+
- @checkstack/secrets-backend@0.3.2
|
|
134
|
+
- @checkstack/status-page-backend@0.4.7
|
|
135
|
+
- @checkstack/sdk@0.125.1
|
|
136
|
+
|
|
137
|
+
## 1.16.0
|
|
138
|
+
|
|
139
|
+
### Minor Changes
|
|
140
|
+
|
|
141
|
+
- c55d7c6: Make collector assertions analyzable: structured per-assertion outcomes on
|
|
142
|
+
every run, pass/fail counts in every aggregate tier, and dedicated analysis
|
|
143
|
+
surfaces. Previously a passing assertion left no trace and only the first
|
|
144
|
+
failure was recorded as a string.
|
|
145
|
+
|
|
146
|
+
- `@checkstack/healthcheck-common` adds the assertion-analytics contract:
|
|
147
|
+
`AssertionOutcomeSchema`, per-bucket `BucketAssertionStats` (stored under
|
|
148
|
+
the platform-owned top-level `assertions` key of `aggregatedResult`), and
|
|
149
|
+
the canonical assertion identity key (`computeAssertionKey` /
|
|
150
|
+
`parseAssertionKey`, a JSON tuple of field/jsonPath/operator/value).
|
|
151
|
+
Editing an assertion starts a new series; identical duplicates collapse.
|
|
152
|
+
- The executor evaluates ALL assertions (no first-failure short-circuit) and
|
|
153
|
+
stores `_assertions` on each collector entry alongside the unchanged
|
|
154
|
+
`_assertionFailed` compatibility string. Pass/fail counts are folded into
|
|
155
|
+
the hourly realtime aggregation, the on-read raw tier, cross-tier bucket
|
|
156
|
+
re-merges, and the daily retention rollup (assertion counts are the only
|
|
157
|
+
`aggregatedResult` content that survives the rollup - they are purely
|
|
158
|
+
additive), so assertion analytics do not silently end at the hourly
|
|
159
|
+
retention horizon.
|
|
160
|
+
- Satellite ingest now evaluates assertions on the core
|
|
161
|
+
(`ingestSatelliteResult`), downgrading a satellite-reported healthy run
|
|
162
|
+
whose assertions fail, and strips ephemeral result fields (e.g. raw HTTP
|
|
163
|
+
bodies) at ingest for parity with local runs. BEHAVIOR CHANGE:
|
|
164
|
+
satellite-executed checks previously never enforced assertions at all;
|
|
165
|
+
they now do, with no satellite upgrade or wire-protocol change. Buffered
|
|
166
|
+
satellite results are evaluated against the configuration current at
|
|
167
|
+
ingest time.
|
|
168
|
+
- The run detail gains an Assertions tab (per-collector groups, pass AND
|
|
169
|
+
fail rows with expected vs actual, a legacy fallback for pre-feature
|
|
170
|
+
runs), and the drawer's auto-chart grid leads each collector group with
|
|
171
|
+
per-assertion pass-rate tiles (sparkline of per-bucket pass rate,
|
|
172
|
+
expandable to a pass/fail StackedTimeline; currently-configured assertions
|
|
173
|
+
appear before any data exists, historical-only series are flagged).
|
|
174
|
+
|
|
175
|
+
State & scale: all new state lives in the existing `healthCheckRuns.result`
|
|
176
|
+
and `healthCheckAggregates.aggregated_result` jsonb columns (durable, shared
|
|
177
|
+
Postgres - no new tables, no pod-local state); reads resolve identically on
|
|
178
|
+
every pod; the run-vs-bucket duplication is the platform's existing
|
|
179
|
+
raw-vs-aggregate tiering with the existing single-writer upsert paths.
|
|
180
|
+
|
|
181
|
+
### Patch Changes
|
|
182
|
+
|
|
183
|
+
- Updated dependencies [c55d7c6]
|
|
184
|
+
- Updated dependencies [c55d7c6]
|
|
185
|
+
- Updated dependencies [a83bcc2]
|
|
186
|
+
- Updated dependencies [c55d7c6]
|
|
187
|
+
- @checkstack/ai-backend@0.10.7
|
|
188
|
+
- @checkstack/healthcheck-common@1.13.0
|
|
189
|
+
- @checkstack/common@0.21.0
|
|
190
|
+
- @checkstack/automation-backend@0.10.9
|
|
191
|
+
- @checkstack/catalog-backend@1.6.7
|
|
192
|
+
- @checkstack/incident-backend@1.9.5
|
|
193
|
+
- @checkstack/satellite-backend@0.8.1
|
|
194
|
+
- @checkstack/backend-api@0.29.1
|
|
195
|
+
- @checkstack/sdk@0.123.1
|
|
196
|
+
- @checkstack/ai-common@0.6.5
|
|
197
|
+
- @checkstack/cache-api@0.3.18
|
|
198
|
+
- @checkstack/catalog-common@2.6.2
|
|
199
|
+
- @checkstack/command-backend@0.2.19
|
|
200
|
+
- @checkstack/gitops-backend@0.5.19
|
|
201
|
+
- @checkstack/gitops-common@0.7.2
|
|
202
|
+
- @checkstack/incident-common@1.7.2
|
|
203
|
+
- @checkstack/maintenance-common@1.8.2
|
|
204
|
+
- @checkstack/notification-common@1.5.2
|
|
205
|
+
- @checkstack/queue-api@0.3.18
|
|
206
|
+
- @checkstack/script-packages-backend@0.3.23
|
|
207
|
+
- @checkstack/secrets-backend@0.3.1
|
|
208
|
+
- @checkstack/secrets-common@0.3.1
|
|
209
|
+
- @checkstack/signal-common@0.2.16
|
|
210
|
+
- @checkstack/status-page-backend@0.4.6
|
|
211
|
+
- @checkstack/status-page-common@0.5.2
|
|
212
|
+
- @checkstack/cache-utils@0.2.23
|
|
213
|
+
|
|
3
214
|
## 1.15.0
|
|
4
215
|
|
|
5
216
|
### Minor Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/healthcheck-backend",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.17.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -14,32 +14,32 @@
|
|
|
14
14
|
"lint:code": "eslint . --max-warnings 0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@checkstack/ai-backend": "0.10.
|
|
18
|
-
"@checkstack/ai-common": "0.6.
|
|
19
|
-
"@checkstack/automation-backend": "0.10.
|
|
20
|
-
"@checkstack/backend-api": "0.
|
|
21
|
-
"@checkstack/cache-api": "0.3.
|
|
22
|
-
"@checkstack/cache-utils": "0.2.
|
|
23
|
-
"@checkstack/catalog-backend": "1.6.
|
|
24
|
-
"@checkstack/catalog-common": "2.6.
|
|
25
|
-
"@checkstack/command-backend": "0.2.
|
|
26
|
-
"@checkstack/common": "0.
|
|
27
|
-
"@checkstack/gitops-backend": "0.5.
|
|
28
|
-
"@checkstack/gitops-common": "0.7.
|
|
29
|
-
"@checkstack/healthcheck-common": "1.
|
|
30
|
-
"@checkstack/incident-backend": "1.
|
|
31
|
-
"@checkstack/incident-common": "1.
|
|
32
|
-
"@checkstack/maintenance-common": "1.8.
|
|
33
|
-
"@checkstack/notification-common": "1.5.
|
|
34
|
-
"@checkstack/queue-api": "0.3.
|
|
35
|
-
"@checkstack/satellite-backend": "0.8.
|
|
36
|
-
"@checkstack/script-packages-backend": "0.3.
|
|
37
|
-
"@checkstack/sdk": "0.
|
|
38
|
-
"@checkstack/secrets-backend": "0.3.
|
|
39
|
-
"@checkstack/secrets-common": "0.3.
|
|
40
|
-
"@checkstack/signal-common": "0.2.
|
|
41
|
-
"@checkstack/status-page-backend": "0.4.
|
|
42
|
-
"@checkstack/status-page-common": "0.5.
|
|
17
|
+
"@checkstack/ai-backend": "0.10.8",
|
|
18
|
+
"@checkstack/ai-common": "0.6.5",
|
|
19
|
+
"@checkstack/automation-backend": "0.10.10",
|
|
20
|
+
"@checkstack/backend-api": "0.30.0",
|
|
21
|
+
"@checkstack/cache-api": "0.3.18",
|
|
22
|
+
"@checkstack/cache-utils": "0.2.23",
|
|
23
|
+
"@checkstack/catalog-backend": "1.6.8",
|
|
24
|
+
"@checkstack/catalog-common": "2.6.2",
|
|
25
|
+
"@checkstack/command-backend": "0.2.20",
|
|
26
|
+
"@checkstack/common": "0.21.0",
|
|
27
|
+
"@checkstack/gitops-backend": "0.5.20",
|
|
28
|
+
"@checkstack/gitops-common": "0.7.2",
|
|
29
|
+
"@checkstack/healthcheck-common": "1.14.0",
|
|
30
|
+
"@checkstack/incident-backend": "1.10.0",
|
|
31
|
+
"@checkstack/incident-common": "1.8.0",
|
|
32
|
+
"@checkstack/maintenance-common": "1.8.2",
|
|
33
|
+
"@checkstack/notification-common": "1.5.2",
|
|
34
|
+
"@checkstack/queue-api": "0.3.18",
|
|
35
|
+
"@checkstack/satellite-backend": "0.8.2",
|
|
36
|
+
"@checkstack/script-packages-backend": "0.3.24",
|
|
37
|
+
"@checkstack/sdk": "0.125.1",
|
|
38
|
+
"@checkstack/secrets-backend": "0.3.2",
|
|
39
|
+
"@checkstack/secrets-common": "0.3.1",
|
|
40
|
+
"@checkstack/signal-common": "0.2.16",
|
|
41
|
+
"@checkstack/status-page-backend": "0.4.7",
|
|
42
|
+
"@checkstack/status-page-common": "0.5.2",
|
|
43
43
|
"@hono/zod-validator": "^0.7.6",
|
|
44
44
|
"@orpc/contract": "^1.14.4",
|
|
45
45
|
"@orpc/server": "^1.14.4",
|
|
@@ -52,8 +52,8 @@
|
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@checkstack/drizzle-helper": "0.0.6",
|
|
55
|
-
"@checkstack/scripts": "0.7.
|
|
56
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
55
|
+
"@checkstack/scripts": "0.7.2",
|
|
56
|
+
"@checkstack/test-utils-backend": "0.1.54",
|
|
57
57
|
"@checkstack/tsconfig": "0.0.7",
|
|
58
58
|
"@types/bun": "^1.0.0",
|
|
59
59
|
"@types/tdigest": "^0.1.5",
|
|
@@ -4,12 +4,14 @@ import {
|
|
|
4
4
|
calculateLatencyStats,
|
|
5
5
|
countStatuses,
|
|
6
6
|
extractLatencies,
|
|
7
|
+
foldRunAssertionStats,
|
|
7
8
|
mergeTieredBuckets,
|
|
8
9
|
combineBuckets,
|
|
9
10
|
reaggregateBuckets,
|
|
10
11
|
mergeAggregatedBucketResults,
|
|
11
12
|
type NormalizedBucket,
|
|
12
13
|
} from "./aggregation-utils";
|
|
14
|
+
import { computeAssertionKey } from "@checkstack/healthcheck-common";
|
|
13
15
|
import {
|
|
14
16
|
VersionedAggregated,
|
|
15
17
|
aggregatedCounter,
|
|
@@ -982,3 +984,133 @@ describe("aggregation-utils", () => {
|
|
|
982
984
|
});
|
|
983
985
|
});
|
|
984
986
|
});
|
|
987
|
+
|
|
988
|
+
describe("assertion stats in aggregation", () => {
|
|
989
|
+
const KEY = computeAssertionKey({
|
|
990
|
+
assertion: { field: "statusCode", operator: "equals", value: 200 },
|
|
991
|
+
});
|
|
992
|
+
|
|
993
|
+
const outcome = (passed: boolean) => ({
|
|
994
|
+
key: KEY,
|
|
995
|
+
field: "statusCode",
|
|
996
|
+
operator: "equals",
|
|
997
|
+
value: "200",
|
|
998
|
+
passed,
|
|
999
|
+
});
|
|
1000
|
+
|
|
1001
|
+
describe("foldRunAssertionStats", () => {
|
|
1002
|
+
it("folds outcomes across a bucket's runs per collector entry", () => {
|
|
1003
|
+
const runs = [
|
|
1004
|
+
{
|
|
1005
|
+
metadata: {
|
|
1006
|
+
collectors: {
|
|
1007
|
+
"uuid-1": { _collectorId: "c", _assertions: [outcome(true)] },
|
|
1008
|
+
},
|
|
1009
|
+
},
|
|
1010
|
+
},
|
|
1011
|
+
{
|
|
1012
|
+
metadata: {
|
|
1013
|
+
collectors: {
|
|
1014
|
+
"uuid-1": { _collectorId: "c", _assertions: [outcome(false)] },
|
|
1015
|
+
},
|
|
1016
|
+
},
|
|
1017
|
+
},
|
|
1018
|
+
// Pre-feature run without outcomes is tolerated.
|
|
1019
|
+
{ metadata: { collectors: { "uuid-1": { _collectorId: "c" } } } },
|
|
1020
|
+
];
|
|
1021
|
+
expect(foldRunAssertionStats(runs)).toEqual({
|
|
1022
|
+
"uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
|
|
1023
|
+
});
|
|
1024
|
+
});
|
|
1025
|
+
|
|
1026
|
+
it("returns undefined when no run carries outcomes", () => {
|
|
1027
|
+
expect(foldRunAssertionStats([{ metadata: {} }, {}])).toBeUndefined();
|
|
1028
|
+
});
|
|
1029
|
+
|
|
1030
|
+
it("ignores malformed outcome entries", () => {
|
|
1031
|
+
const runs = [
|
|
1032
|
+
{
|
|
1033
|
+
metadata: {
|
|
1034
|
+
collectors: {
|
|
1035
|
+
"uuid-1": {
|
|
1036
|
+
_collectorId: "c",
|
|
1037
|
+
_assertions: ["garbage", outcome(true)],
|
|
1038
|
+
},
|
|
1039
|
+
},
|
|
1040
|
+
},
|
|
1041
|
+
},
|
|
1042
|
+
];
|
|
1043
|
+
expect(foldRunAssertionStats(runs)).toEqual({
|
|
1044
|
+
"uuid-1": { [KEY]: { passCount: 1, failCount: 0 } },
|
|
1045
|
+
});
|
|
1046
|
+
});
|
|
1047
|
+
});
|
|
1048
|
+
|
|
1049
|
+
describe("mergeAggregatedBucketResults with assertion stats", () => {
|
|
1050
|
+
it("merges counts additively and keeps them out of strategy merging", () => {
|
|
1051
|
+
const mergeAggregatedStates = mock(
|
|
1052
|
+
(a: Record<string, unknown>, b: Record<string, unknown>) => ({
|
|
1053
|
+
...a,
|
|
1054
|
+
...b,
|
|
1055
|
+
}),
|
|
1056
|
+
);
|
|
1057
|
+
const registry = {
|
|
1058
|
+
getStrategy: mock(() => ({
|
|
1059
|
+
aggregatedResult: { mergeAggregatedStates },
|
|
1060
|
+
})),
|
|
1061
|
+
register: mock(() => {}),
|
|
1062
|
+
getStrategies: mock(() => []),
|
|
1063
|
+
getStrategiesWithMeta: mock(() => []),
|
|
1064
|
+
} as unknown as HealthCheckRegistry;
|
|
1065
|
+
const collectorRegistry = {
|
|
1066
|
+
register: mock(() => {}),
|
|
1067
|
+
getCollector: mock(() => undefined),
|
|
1068
|
+
getCollectors: mock(() => []),
|
|
1069
|
+
} as unknown as CollectorRegistry;
|
|
1070
|
+
|
|
1071
|
+
const merged = mergeAggregatedBucketResults({
|
|
1072
|
+
aggregatedResults: [
|
|
1073
|
+
{
|
|
1074
|
+
uptime: 1,
|
|
1075
|
+
assertions: { "uuid-1": { [KEY]: { passCount: 3, failCount: 1 } } },
|
|
1076
|
+
},
|
|
1077
|
+
{
|
|
1078
|
+
uptime: 2,
|
|
1079
|
+
assertions: { "uuid-1": { [KEY]: { passCount: 2, failCount: 0 } } },
|
|
1080
|
+
},
|
|
1081
|
+
],
|
|
1082
|
+
collectorRegistry,
|
|
1083
|
+
registry,
|
|
1084
|
+
strategyId: "test-strategy",
|
|
1085
|
+
});
|
|
1086
|
+
|
|
1087
|
+
expect(merged?.assertions).toEqual({
|
|
1088
|
+
"uuid-1": { [KEY]: { passCount: 5, failCount: 1 } },
|
|
1089
|
+
});
|
|
1090
|
+
// The strategy merger only ever saw its own fields.
|
|
1091
|
+
for (const call of mergeAggregatedStates.mock.calls) {
|
|
1092
|
+
for (const arg of call) {
|
|
1093
|
+
expect(
|
|
1094
|
+
(arg as Record<string, unknown>).assertions,
|
|
1095
|
+
).toBeUndefined();
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
|
|
1100
|
+
it("single-bucket results pass through with their stats intact", () => {
|
|
1101
|
+
const { collectorRegistry, registry, strategyId } =
|
|
1102
|
+
createMockRegistries();
|
|
1103
|
+
const only = {
|
|
1104
|
+
assertions: { "uuid-1": { [KEY]: { passCount: 7, failCount: 0 } } },
|
|
1105
|
+
};
|
|
1106
|
+
expect(
|
|
1107
|
+
mergeAggregatedBucketResults({
|
|
1108
|
+
aggregatedResults: [only],
|
|
1109
|
+
collectorRegistry,
|
|
1110
|
+
registry,
|
|
1111
|
+
strategyId,
|
|
1112
|
+
}),
|
|
1113
|
+
).toBe(only);
|
|
1114
|
+
});
|
|
1115
|
+
});
|
|
1116
|
+
});
|
package/src/aggregation-utils.ts
CHANGED
|
@@ -2,6 +2,15 @@ import type {
|
|
|
2
2
|
CollectorRegistry,
|
|
3
3
|
HealthCheckRegistry,
|
|
4
4
|
} from "@checkstack/backend-api";
|
|
5
|
+
import {
|
|
6
|
+
ASSERTIONS_AGG_KEY,
|
|
7
|
+
AssertionOutcomeSchema,
|
|
8
|
+
foldOutcomesIntoStats,
|
|
9
|
+
mergeAssertionStats,
|
|
10
|
+
readAssertionStats,
|
|
11
|
+
type AssertionOutcome,
|
|
12
|
+
type BucketAssertionStats,
|
|
13
|
+
} from "@checkstack/healthcheck-common";
|
|
5
14
|
|
|
6
15
|
// ===== Percentile Calculation =====
|
|
7
16
|
|
|
@@ -138,7 +147,8 @@ export function aggregateCollectorData(
|
|
|
138
147
|
const existing = aggregatedByUuid.get(uuid)?.aggregated;
|
|
139
148
|
|
|
140
149
|
// Strip internal fields from collector data
|
|
141
|
-
const { _collectorId, _assertionFailed, ...collectorMetadata } =
|
|
150
|
+
const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
|
|
151
|
+
data;
|
|
142
152
|
|
|
143
153
|
// Call mergeResult to incrementally aggregate
|
|
144
154
|
const merged = registered.collector.mergeResult(existing, {
|
|
@@ -166,6 +176,39 @@ export function aggregateCollectorData(
|
|
|
166
176
|
return result;
|
|
167
177
|
}
|
|
168
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Fold the structured assertion outcomes of a bucket's raw runs into the
|
|
181
|
+
* per-assertion pass/fail stats (the on-read raw tier's counterpart of the
|
|
182
|
+
* realtime hourly fold). Returns undefined when no run carries outcomes.
|
|
183
|
+
*/
|
|
184
|
+
export function foldRunAssertionStats(
|
|
185
|
+
runs: Array<{ metadata?: Record<string, unknown> }>,
|
|
186
|
+
): BucketAssertionStats | undefined {
|
|
187
|
+
let stats: BucketAssertionStats | undefined;
|
|
188
|
+
for (const run of runs) {
|
|
189
|
+
const collectors = run.metadata?.collectors as
|
|
190
|
+
| Record<string, Record<string, unknown>>
|
|
191
|
+
| undefined;
|
|
192
|
+
if (!collectors) continue;
|
|
193
|
+
for (const [uuid, data] of Object.entries(collectors)) {
|
|
194
|
+
const rawOutcomes = data._assertions;
|
|
195
|
+
if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
|
|
196
|
+
const outcomes: AssertionOutcome[] = [];
|
|
197
|
+
for (const raw of rawOutcomes) {
|
|
198
|
+
const parsed = AssertionOutcomeSchema.safeParse(raw);
|
|
199
|
+
if (parsed.success) outcomes.push(parsed.data);
|
|
200
|
+
}
|
|
201
|
+
if (outcomes.length === 0) continue;
|
|
202
|
+
stats = foldOutcomesIntoStats({
|
|
203
|
+
stats,
|
|
204
|
+
collectorEntryId: uuid,
|
|
205
|
+
outcomes,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return stats;
|
|
210
|
+
}
|
|
211
|
+
|
|
169
212
|
// ===== Bucket Result Merging =====
|
|
170
213
|
|
|
171
214
|
/**
|
|
@@ -201,15 +244,29 @@ export function mergeAggregatedBucketResults(params: {
|
|
|
201
244
|
return validResults[0];
|
|
202
245
|
}
|
|
203
246
|
|
|
247
|
+
// === Platform-level assertion stats (additive, never strategy-merged) ===
|
|
248
|
+
let mergedAssertionStats: BucketAssertionStats | undefined;
|
|
249
|
+
for (const result of validResults) {
|
|
250
|
+
mergedAssertionStats = mergeAssertionStats({
|
|
251
|
+
a: mergedAssertionStats,
|
|
252
|
+
b: readAssertionStats({ aggregatedResult: result }),
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
|
|
204
256
|
// === Strategy-level field merging ===
|
|
205
257
|
let mergedStrategyFields: Record<string, unknown> = {};
|
|
206
258
|
|
|
207
259
|
const registeredStrategy = registry.getStrategy(strategyId);
|
|
208
260
|
if (registeredStrategy?.aggregatedResult) {
|
|
209
|
-
// Extract strategy-level fields (everything except
|
|
261
|
+
// Extract strategy-level fields (everything except the platform-owned
|
|
262
|
+
// 'collectors' and 'assertions' keys)
|
|
210
263
|
const strategyDataSets: Array<Record<string, unknown>> = [];
|
|
211
264
|
for (const result of validResults) {
|
|
212
|
-
const {
|
|
265
|
+
const {
|
|
266
|
+
collectors: _collectors,
|
|
267
|
+
[ASSERTIONS_AGG_KEY]: _assertions,
|
|
268
|
+
...strategyFields
|
|
269
|
+
} = result;
|
|
213
270
|
if (Object.keys(strategyFields).length > 0) {
|
|
214
271
|
strategyDataSets.push(strategyFields);
|
|
215
272
|
}
|
|
@@ -233,7 +290,11 @@ export function mergeAggregatedBucketResults(params: {
|
|
|
233
290
|
}
|
|
234
291
|
} else {
|
|
235
292
|
// Strategy not found - preserve strategy fields from first result
|
|
236
|
-
const {
|
|
293
|
+
const {
|
|
294
|
+
collectors: _collectors,
|
|
295
|
+
[ASSERTIONS_AGG_KEY]: _assertions,
|
|
296
|
+
...firstStrategyFields
|
|
297
|
+
} = validResults[0];
|
|
237
298
|
mergedStrategyFields = firstStrategyFields;
|
|
238
299
|
}
|
|
239
300
|
|
|
@@ -302,17 +363,20 @@ export function mergeAggregatedBucketResults(params: {
|
|
|
302
363
|
}
|
|
303
364
|
}
|
|
304
365
|
|
|
305
|
-
// Combine strategy fields and
|
|
366
|
+
// Combine strategy fields, collector fields, and assertion stats
|
|
306
367
|
const hasCollectors = Object.keys(mergedCollectors).length > 0;
|
|
307
368
|
const hasStrategyFields = Object.keys(mergedStrategyFields).length > 0;
|
|
308
369
|
|
|
309
|
-
if (!hasCollectors && !hasStrategyFields) {
|
|
370
|
+
if (!hasCollectors && !hasStrategyFields && mergedAssertionStats === undefined) {
|
|
310
371
|
return undefined;
|
|
311
372
|
}
|
|
312
373
|
|
|
313
374
|
return {
|
|
314
375
|
...mergedStrategyFields,
|
|
315
376
|
...(hasCollectors ? { collectors: mergedCollectors } : {}),
|
|
377
|
+
...(mergedAssertionStats === undefined
|
|
378
|
+
? {}
|
|
379
|
+
: { [ASSERTIONS_AGG_KEY]: mergedAssertionStats }),
|
|
316
380
|
};
|
|
317
381
|
}
|
|
318
382
|
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
2
|
import type { CollectorAssertion } from "@checkstack/healthcheck-common";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
ASSERTION_ACTUAL_MAX_LENGTH,
|
|
5
|
+
computeAssertionKey,
|
|
6
|
+
} from "@checkstack/healthcheck-common";
|
|
7
|
+
import {
|
|
8
|
+
evaluateCollectorAssertionOutcomes,
|
|
9
|
+
evaluateCollectorAssertions,
|
|
10
|
+
} from "./collector-assertions";
|
|
4
11
|
|
|
5
12
|
/** The HTTP Request collector's result shape, as the executor sees it. */
|
|
6
13
|
const httpResult = (body: string): Record<string, unknown> => ({
|
|
@@ -274,3 +281,92 @@ describe("evaluateCollectorAssertions", () => {
|
|
|
274
281
|
});
|
|
275
282
|
});
|
|
276
283
|
});
|
|
284
|
+
|
|
285
|
+
describe("evaluateCollectorAssertionOutcomes", () => {
|
|
286
|
+
it("evaluates ALL assertions instead of short-circuiting on failure", () => {
|
|
287
|
+
const result = httpResult('{"status":"degraded"}');
|
|
288
|
+
const { outcomes, firstFailureMessage } =
|
|
289
|
+
evaluateCollectorAssertionOutcomes({
|
|
290
|
+
assertions: [
|
|
291
|
+
{ field: "statusCode", operator: "equals", value: 500 }, // fails
|
|
292
|
+
{ field: "success", operator: "isTrue" }, // passes
|
|
293
|
+
jsonPathAssertion("$.status", "equals", "ok"), // fails
|
|
294
|
+
],
|
|
295
|
+
result,
|
|
296
|
+
});
|
|
297
|
+
expect(outcomes.length).toBe(3);
|
|
298
|
+
expect(outcomes.map((o) => o.passed)).toEqual([false, true, false]);
|
|
299
|
+
// The legacy string still reports the FIRST failure only.
|
|
300
|
+
expect(firstFailureMessage).toBe("statusCode equals 500");
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
it("captures observed values for plain and JSONPath assertions", () => {
|
|
304
|
+
const result = httpResult('{"status":"ok","items":[1,2]}');
|
|
305
|
+
const { outcomes } = evaluateCollectorAssertionOutcomes({
|
|
306
|
+
assertions: [
|
|
307
|
+
{ field: "statusCode", operator: "equals", value: 200 },
|
|
308
|
+
jsonPathAssertion("$.status", "equals", "ok"),
|
|
309
|
+
jsonPathAssertion("$.items", "lengthEquals", 2),
|
|
310
|
+
],
|
|
311
|
+
result,
|
|
312
|
+
});
|
|
313
|
+
expect(outcomes[0].actual).toBe("200");
|
|
314
|
+
expect(outcomes[1].actual).toBe("ok");
|
|
315
|
+
expect(outcomes[2].actual).toBe("[1,2]");
|
|
316
|
+
expect(outcomes.every((o) => o.passed)).toBe(true);
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
it("truncates long observed values", () => {
|
|
320
|
+
const longBody = JSON.stringify({ blob: "x".repeat(5000) });
|
|
321
|
+
const { outcomes } = evaluateCollectorAssertionOutcomes({
|
|
322
|
+
assertions: [
|
|
323
|
+
{ field: "body", operator: "contains", value: "x" },
|
|
324
|
+
],
|
|
325
|
+
result: httpResult(longBody),
|
|
326
|
+
});
|
|
327
|
+
expect(outcomes[0].passed).toBe(true);
|
|
328
|
+
expect(outcomes[0].actual?.length).toBe(ASSERTION_ACTUAL_MAX_LENGTH);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
it("fail-closed diagnostics become failed outcomes with messages", () => {
|
|
332
|
+
const { outcomes, firstFailureMessage } =
|
|
333
|
+
evaluateCollectorAssertionOutcomes({
|
|
334
|
+
assertions: [
|
|
335
|
+
{ field: "body.$", jsonPath: " ", operator: "exists" },
|
|
336
|
+
jsonPathAssertion("$.status", "equals", "ok"),
|
|
337
|
+
],
|
|
338
|
+
result: httpResult("not json"),
|
|
339
|
+
});
|
|
340
|
+
expect(outcomes[0].passed).toBe(false);
|
|
341
|
+
expect(outcomes[0].message).toBe("missing JSONPath expression");
|
|
342
|
+
expect(outcomes[1].passed).toBe(false);
|
|
343
|
+
expect(outcomes[1].message).toBe('field "body" is not valid JSON');
|
|
344
|
+
expect(firstFailureMessage).toBe(
|
|
345
|
+
"body.$ exists (missing JSONPath expression)",
|
|
346
|
+
);
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it("outcomes carry the canonical assertion identity key", () => {
|
|
350
|
+
const assertion: CollectorAssertion = {
|
|
351
|
+
field: "statusCode",
|
|
352
|
+
operator: "equals",
|
|
353
|
+
value: 200,
|
|
354
|
+
};
|
|
355
|
+
const { outcomes } = evaluateCollectorAssertionOutcomes({
|
|
356
|
+
assertions: [assertion],
|
|
357
|
+
result: httpResult("{}"),
|
|
358
|
+
});
|
|
359
|
+
expect(outcomes[0].key).toBe(computeAssertionKey({ assertion }));
|
|
360
|
+
expect(outcomes[0].value).toBe("200");
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it("no assertions yields an empty evaluation", () => {
|
|
364
|
+
const { outcomes, firstFailureMessage } =
|
|
365
|
+
evaluateCollectorAssertionOutcomes({
|
|
366
|
+
assertions: undefined,
|
|
367
|
+
result: httpResult("{}"),
|
|
368
|
+
});
|
|
369
|
+
expect(outcomes).toEqual([]);
|
|
370
|
+
expect(firstFailureMessage).toBeUndefined();
|
|
371
|
+
});
|
|
372
|
+
});
|