@checkstack/healthcheck-backend 1.21.3 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +219 -0
- package/drizzle/0021_amazing_wolf_cub.sql +8 -0
- package/drizzle/meta/0021_snapshot.json +717 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +21 -20
- package/src/ai/system-signals-contributor.test.ts +1 -0
- package/src/cache.test.ts +3 -0
- package/src/effective-environments.test.ts +63 -2
- package/src/effective-environments.ts +34 -0
- package/src/health-entity.ts +8 -2
- package/src/health-state.ts +15 -6
- package/src/index.ts +23 -0
- package/src/queue-executor.ts +545 -565
- package/src/router-satellite-ingest.test.ts +136 -0
- package/src/router.ts +65 -4
- package/src/schema.ts +21 -0
- package/src/service-batching.test.ts +3 -1
- package/src/service-ingest-assertions.test.ts +33 -60
- package/src/service-paused-filter.test.ts +9 -4
- package/src/service-rollup-worst-wins.test.ts +147 -55
- package/src/service.ts +355 -148
- package/src/state-evaluator.test.ts +49 -0
- package/src/system-health-override.ts +6 -1
- package/tsconfig.json +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/healthcheck-backend",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.22.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -14,32 +14,33 @@
|
|
|
14
14
|
"lint:code": "eslint . --max-warnings 0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@checkstack/ai-backend": "0.11.
|
|
17
|
+
"@checkstack/ai-backend": "0.11.4",
|
|
18
18
|
"@checkstack/ai-common": "0.6.7",
|
|
19
|
-
"@checkstack/automation-backend": "0.11.
|
|
20
|
-
"@checkstack/backend-api": "0.34.
|
|
19
|
+
"@checkstack/automation-backend": "0.11.8",
|
|
20
|
+
"@checkstack/backend-api": "0.34.1",
|
|
21
21
|
"@checkstack/cache-api": "0.3.20",
|
|
22
22
|
"@checkstack/cache-utils": "0.3.1",
|
|
23
|
-
"@checkstack/catalog-backend": "1.10.
|
|
24
|
-
"@checkstack/catalog-common": "2.8.
|
|
25
|
-
"@checkstack/command-backend": "0.2.
|
|
23
|
+
"@checkstack/catalog-backend": "1.10.1",
|
|
24
|
+
"@checkstack/catalog-common": "2.8.1",
|
|
25
|
+
"@checkstack/command-backend": "0.2.27",
|
|
26
26
|
"@checkstack/common": "0.23.0",
|
|
27
|
-
"@checkstack/gitops-backend": "0.5.
|
|
27
|
+
"@checkstack/gitops-backend": "0.5.27",
|
|
28
28
|
"@checkstack/gitops-common": "0.7.4",
|
|
29
|
-
"@checkstack/healthcheck-common": "1.
|
|
30
|
-
"@checkstack/
|
|
31
|
-
"@checkstack/incident-
|
|
32
|
-
"@checkstack/
|
|
33
|
-
"@checkstack/
|
|
29
|
+
"@checkstack/healthcheck-common": "1.19.0",
|
|
30
|
+
"@checkstack/healthcheck-execution": "0.35.0",
|
|
31
|
+
"@checkstack/incident-backend": "1.13.6",
|
|
32
|
+
"@checkstack/incident-common": "1.10.5",
|
|
33
|
+
"@checkstack/maintenance-common": "1.10.5",
|
|
34
|
+
"@checkstack/notification-common": "1.8.0",
|
|
34
35
|
"@checkstack/queue-api": "0.4.0",
|
|
35
|
-
"@checkstack/satellite-backend": "0.9.
|
|
36
|
-
"@checkstack/script-packages-backend": "0.4.
|
|
37
|
-
"@checkstack/sdk": "0.
|
|
38
|
-
"@checkstack/secrets-backend": "0.3.
|
|
36
|
+
"@checkstack/satellite-backend": "0.9.4",
|
|
37
|
+
"@checkstack/script-packages-backend": "0.4.6",
|
|
38
|
+
"@checkstack/sdk": "0.135.1",
|
|
39
|
+
"@checkstack/secrets-backend": "0.3.9",
|
|
39
40
|
"@checkstack/secrets-common": "0.3.3",
|
|
40
41
|
"@checkstack/signal-common": "0.3.1",
|
|
41
|
-
"@checkstack/status-page-backend": "0.6.
|
|
42
|
-
"@checkstack/status-page-common": "0.6.
|
|
42
|
+
"@checkstack/status-page-backend": "0.6.6",
|
|
43
|
+
"@checkstack/status-page-common": "0.6.5",
|
|
43
44
|
"@hono/zod-validator": "^0.7.6",
|
|
44
45
|
"@orpc/contract": "^1.14.4",
|
|
45
46
|
"@orpc/server": "^1.14.4",
|
|
@@ -52,7 +53,7 @@
|
|
|
52
53
|
"devDependencies": {
|
|
53
54
|
"@checkstack/drizzle-helper": "0.0.6",
|
|
54
55
|
"@checkstack/scripts": "0.7.6",
|
|
55
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
56
|
+
"@checkstack/test-utils-backend": "0.1.61",
|
|
56
57
|
"@checkstack/tsconfig": "0.0.7",
|
|
57
58
|
"@types/bun": "^1.0.0",
|
|
58
59
|
"@types/pg": "^8.20.0",
|
package/src/cache.test.ts
CHANGED
|
@@ -63,6 +63,9 @@ function response(
|
|
|
63
63
|
runsConsidered: 1,
|
|
64
64
|
sliceCount: c.sliceCount ?? 1,
|
|
65
65
|
failingSliceCount: c.failingSliceCount ?? 0,
|
|
66
|
+
// These tests exercise the cache's fingerprint/invalidation behaviour,
|
|
67
|
+
// which reads the counts rather than the per-slice breakdown.
|
|
68
|
+
slices: [],
|
|
66
69
|
})),
|
|
67
70
|
};
|
|
68
71
|
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import { describe, it, expect } from "bun:test";
|
|
2
|
-
import {
|
|
1
|
+
import { describe, it, test, expect } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
resolveEffectiveEnvironments,
|
|
4
|
+
resolveSatelliteEnvironments,
|
|
5
|
+
} from "./effective-environments";
|
|
3
6
|
import type { Environment } from "@checkstack/catalog-common";
|
|
4
7
|
|
|
5
8
|
const env = (
|
|
@@ -91,3 +94,61 @@ describe("resolveEffectiveEnvironments", () => {
|
|
|
91
94
|
expect(result).toEqual([]);
|
|
92
95
|
});
|
|
93
96
|
});
|
|
97
|
+
|
|
98
|
+
describe("resolveSatelliteEnvironments", () => {
|
|
99
|
+
const effective = [
|
|
100
|
+
{ id: "env-prod", name: "Production", fields: {} },
|
|
101
|
+
{ id: "env-stage", name: "Staging", fields: {} },
|
|
102
|
+
];
|
|
103
|
+
|
|
104
|
+
test("an unscoped satellite runs every environment the assignment resolved to", () => {
|
|
105
|
+
// The backfill-free default: a NULL column means no scoping, so existing
|
|
106
|
+
// assignments keep behaving exactly as they did.
|
|
107
|
+
expect(
|
|
108
|
+
resolveSatelliteEnvironments({
|
|
109
|
+
effective,
|
|
110
|
+
satelliteEnvironmentIds: undefined,
|
|
111
|
+
}),
|
|
112
|
+
).toEqual(effective);
|
|
113
|
+
expect(
|
|
114
|
+
resolveSatelliteEnvironments({ effective, satelliteEnvironmentIds: null }),
|
|
115
|
+
).toEqual(effective);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("a scoped satellite runs only its own environments", () => {
|
|
119
|
+
// The point of the feature: the prod satellite probes prod, and never
|
|
120
|
+
// reaches for a staging endpoint it may have no route to.
|
|
121
|
+
expect(
|
|
122
|
+
resolveSatelliteEnvironments({
|
|
123
|
+
effective,
|
|
124
|
+
satelliteEnvironmentIds: ["env-prod"],
|
|
125
|
+
}).map((e) => e.id),
|
|
126
|
+
).toEqual(["env-prod"]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a satellite can NARROW but never widen the assignment's scope", () => {
|
|
130
|
+
// An id the assignment does not cover silently drops - a satellite must not
|
|
131
|
+
// be able to probe an environment the assignment itself excluded.
|
|
132
|
+
expect(
|
|
133
|
+
resolveSatelliteEnvironments({
|
|
134
|
+
effective,
|
|
135
|
+
satelliteEnvironmentIds: ["env-prod", "env-secret"],
|
|
136
|
+
}).map((e) => e.id),
|
|
137
|
+
).toEqual(["env-prod"]);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("an empty selector opts the satellite out into a single env-less run", () => {
|
|
141
|
+
expect(
|
|
142
|
+
resolveSatelliteEnvironments({ effective, satelliteEnvironmentIds: [] }),
|
|
143
|
+
).toEqual([]);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("preserves the assignment's order, so fan-out stays deterministic", () => {
|
|
147
|
+
expect(
|
|
148
|
+
resolveSatelliteEnvironments({
|
|
149
|
+
effective,
|
|
150
|
+
satelliteEnvironmentIds: ["env-stage", "env-prod"],
|
|
151
|
+
}).map((e) => e.id),
|
|
152
|
+
).toEqual(["env-prod", "env-stage"]);
|
|
153
|
+
});
|
|
154
|
+
});
|
|
@@ -62,3 +62,37 @@ export function resolveEffectiveEnvironments({
|
|
|
62
62
|
.filter((env) => wanted.has(env.id))
|
|
63
63
|
.map((env) => toEffective(env));
|
|
64
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Narrow an assignment's effective environments to those a SPECIFIC satellite
|
|
68
|
+
* should run.
|
|
69
|
+
*
|
|
70
|
+
* A satellite's selector can only ever narrow, never widen: the assignment
|
|
71
|
+
* decides which environments exist for this check, and the satellite decides
|
|
72
|
+
* which of those it is responsible for. That ordering is what lets a prod
|
|
73
|
+
* satellite run only prod without being able to reach environments the
|
|
74
|
+
* assignment itself excluded.
|
|
75
|
+
*
|
|
76
|
+
* - `undefined` / `null` => every environment the assignment resolved to. This
|
|
77
|
+
* is the default for an unscoped satellite, so existing assignments keep
|
|
78
|
+
* behaving exactly as they did.
|
|
79
|
+
* - `[]` => opt out: one env-less run on this satellite.
|
|
80
|
+
* - non-empty => exactly those ids, intersected with `effective`;
|
|
81
|
+
* an id the assignment does not cover silently drops.
|
|
82
|
+
*
|
|
83
|
+
* Order follows `effective`, so fan-out order stays deterministic.
|
|
84
|
+
*/
|
|
85
|
+
export function resolveSatelliteEnvironments({
|
|
86
|
+
effective,
|
|
87
|
+
satelliteEnvironmentIds,
|
|
88
|
+
}: {
|
|
89
|
+
effective: EffectiveEnvironment[];
|
|
90
|
+
satelliteEnvironmentIds: string[] | null | undefined;
|
|
91
|
+
}): EffectiveEnvironment[] {
|
|
92
|
+
if (satelliteEnvironmentIds === null || satelliteEnvironmentIds === undefined) {
|
|
93
|
+
return effective;
|
|
94
|
+
}
|
|
95
|
+
if (satelliteEnvironmentIds.length === 0) return [];
|
|
96
|
+
const wanted = new Set(satelliteEnvironmentIds);
|
|
97
|
+
return effective.filter((env) => wanted.has(env.id));
|
|
98
|
+
}
|
package/src/health-entity.ts
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* - the `writeHealthEntity` helper called at every evaluation-write site.
|
|
23
23
|
*/
|
|
24
24
|
import { z } from "zod";
|
|
25
|
-
import {
|
|
25
|
+
import { SystemHealthStatusSchema } from "@checkstack/healthcheck-common";
|
|
26
26
|
import type { AdvisoryLockService } from "@checkstack/backend-api";
|
|
27
27
|
import type {
|
|
28
28
|
EntityChangeDeriver,
|
|
@@ -43,7 +43,13 @@ export const HEALTH_ENTITY_KIND = "health";
|
|
|
43
43
|
* same durable data `getSystemHealthStatus` reads — never materialized.
|
|
44
44
|
*/
|
|
45
45
|
export const HealthEntityStateSchema = z.object({
|
|
46
|
-
|
|
46
|
+
/**
|
|
47
|
+
* `unknown` when the system's checks have produced no runs to evaluate.
|
|
48
|
+
* Automations matching on `unhealthy` therefore do NOT fire for a system that
|
|
49
|
+
* is merely unmeasured - which is the point: an unmeasured system is not a
|
|
50
|
+
* detected outage, and inventing `healthy` for it would hide the gap instead.
|
|
51
|
+
*/
|
|
52
|
+
status: SystemHealthStatusSchema,
|
|
47
53
|
healthyChecks: z.number().int().nonnegative(),
|
|
48
54
|
totalChecks: z.number().int().nonnegative(),
|
|
49
55
|
});
|
package/src/health-state.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { and, desc, eq, gte, isNull } from "drizzle-orm";
|
|
2
|
-
import type {
|
|
2
|
+
import type { SystemHealthStatus } from "@checkstack/healthcheck-common";
|
|
3
3
|
import type { Logger, SafeDatabase } from "@checkstack/backend-api";
|
|
4
4
|
import type { InferClient } from "@checkstack/common";
|
|
5
5
|
import { MaintenanceApi } from "@checkstack/maintenance-common";
|
|
@@ -20,8 +20,11 @@ type MaintenanceClient = InferClient<typeof MaintenanceApi>;
|
|
|
20
20
|
* re-deriving the math each time.
|
|
21
21
|
*/
|
|
22
22
|
export interface HealthState {
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Aggregate status across all enabled checks. `unknown` when nothing has been
|
|
25
|
+
* measured yet - the system has no checks, or none of them has ever run.
|
|
26
|
+
*/
|
|
27
|
+
status: SystemHealthStatus;
|
|
25
28
|
/**
|
|
26
29
|
* When the system most recently entered `status`. Null when no
|
|
27
30
|
* transition has been recorded yet (fail-safe: never throws).
|
|
@@ -58,7 +61,8 @@ export interface HealthState {
|
|
|
58
61
|
|
|
59
62
|
/** Raw inputs to the pure builder, decoupled from the DB layer. */
|
|
60
63
|
export interface HealthStateInputs {
|
|
61
|
-
|
|
64
|
+
/** May be `unknown` - nothing measured yet. */
|
|
65
|
+
status: SystemHealthStatus;
|
|
62
66
|
inStatusSince: Date | null;
|
|
63
67
|
latencyMs?: number;
|
|
64
68
|
avgLatencyMs?: number;
|
|
@@ -330,7 +334,7 @@ export async function computeHealthState({
|
|
|
330
334
|
*/
|
|
331
335
|
environmentId?: string | null;
|
|
332
336
|
/** Returns the aggregate status for the system (per-check when scoped). */
|
|
333
|
-
resolveStatus: () => Promise<
|
|
337
|
+
resolveStatus: () => Promise<SystemHealthStatus>;
|
|
334
338
|
maintenanceClient?: MaintenanceClient;
|
|
335
339
|
logger?: Logger;
|
|
336
340
|
/** Trailing window (minutes) for the transition count. */
|
|
@@ -341,7 +345,12 @@ export async function computeHealthState({
|
|
|
341
345
|
|
|
342
346
|
const [inStatusSince, latest, windowed, inMaintenance, transitionsInWindow] =
|
|
343
347
|
await Promise.all([
|
|
344
|
-
|
|
348
|
+
// An UNMEASURED system has no transition rows to find - and `unknown` is
|
|
349
|
+
// not a member of the transition table's status enum, so querying for it
|
|
350
|
+
// would be a database error, not merely an empty result.
|
|
351
|
+
status === "unknown"
|
|
352
|
+
? Promise.resolve(null)
|
|
353
|
+
: findInStatusSince({ db, systemId, status, environmentId }),
|
|
345
354
|
findLatestRun({ db, systemId, configurationId, environmentId }),
|
|
346
355
|
computeWindowedMetrics({ db, systemId, configurationId, environmentId, now }),
|
|
347
356
|
resolveInMaintenance({ maintenanceClient, systemId, logger }),
|
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
setupHealthCheckWorker,
|
|
3
3
|
recomputeSystemRollupHealth,
|
|
4
|
+
persistRunAndReact,
|
|
4
5
|
} from "./queue-executor";
|
|
5
6
|
import { reconcileHealthCheckJobs } from "./schedule-reconciler";
|
|
6
7
|
import { setupRetentionJob } from "./retention-job";
|
|
@@ -592,6 +593,28 @@ export default createBackendPlugin({
|
|
|
592
593
|
logger,
|
|
593
594
|
});
|
|
594
595
|
},
|
|
596
|
+
// Bind the service deps ONCE so an ingested satellite result drives
|
|
597
|
+
// the exact same post-run path (`persistRunAndReact`) as a local run:
|
|
598
|
+
// reactive entity write, cache reconcile, signals, automation hooks,
|
|
599
|
+
// transition, and notification. The router only supplies the per-run
|
|
600
|
+
// fields, so the two callers cannot drift.
|
|
601
|
+
reactToSatelliteRun: (run) =>
|
|
602
|
+
persistRunAndReact({
|
|
603
|
+
db: database,
|
|
604
|
+
service,
|
|
605
|
+
cache,
|
|
606
|
+
signalService,
|
|
607
|
+
notificationClient,
|
|
608
|
+
catalogClient,
|
|
609
|
+
maintenanceClient,
|
|
610
|
+
incidentClient,
|
|
611
|
+
getHealthEntity: () => healthEntity,
|
|
612
|
+
getEmitHook: () => storedEmitHook,
|
|
613
|
+
collectorRegistry,
|
|
614
|
+
advisoryLock,
|
|
615
|
+
logger,
|
|
616
|
+
...run,
|
|
617
|
+
}),
|
|
595
618
|
});
|
|
596
619
|
rpc.registerRouter(healthCheckRouter, healthCheckContract);
|
|
597
620
|
|