@checkstack/healthcheck-backend 1.21.2 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +289 -0
- package/drizzle/0021_amazing_wolf_cub.sql +8 -0
- package/drizzle/meta/0021_snapshot.json +717 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +30 -31
- package/src/ai/system-signals-contributor.test.ts +1 -0
- package/src/cache.test.ts +3 -0
- package/src/effective-environments.test.ts +63 -2
- package/src/effective-environments.ts +34 -0
- package/src/health-entity.ts +8 -2
- package/src/health-state.ts +15 -6
- package/src/index.ts +53 -14
- package/src/queue-executor.ts +554 -578
- package/src/realtime-aggregation.test.ts +10 -11
- package/src/realtime-aggregation.ts +12 -50
- package/src/router-satellite-ingest.test.ts +136 -0
- package/src/router.ts +65 -4
- package/src/schema.ts +21 -0
- package/src/service-batching.test.ts +3 -1
- package/src/service-ingest-assertions.test.ts +33 -60
- package/src/service-paused-filter.test.ts +9 -4
- package/src/service-rollup-worst-wins.test.ts +147 -55
- package/src/service.ts +355 -148
- package/src/state-evaluator.test.ts +49 -0
- package/src/system-health-override.ts +6 -1
- package/tsconfig.json +3 -0
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import { describe, it, expect, mock, beforeEach } from "bun:test";
|
|
2
|
+
import { getHourBucketStart, incrementHourlyAggregate } from "./realtime-aggregation";
|
|
2
3
|
import {
|
|
3
|
-
getHourBucketStart,
|
|
4
4
|
serializeTDigest,
|
|
5
5
|
deserializeTDigest,
|
|
6
|
-
|
|
7
|
-
} from "
|
|
6
|
+
createTDigest,
|
|
7
|
+
} from "@checkstack/backend-api";
|
|
8
8
|
import { computeAssertionKey } from "@checkstack/healthcheck-common";
|
|
9
|
-
import { TDigest } from "tdigest";
|
|
10
9
|
|
|
11
10
|
describe("getHourBucketStart", () => {
|
|
12
11
|
it("floors timestamp to the hour", () => {
|
|
@@ -42,7 +41,7 @@ describe("getHourBucketStart", () => {
|
|
|
42
41
|
|
|
43
42
|
describe("t-digest serialization", () => {
|
|
44
43
|
it("serializes and deserializes empty t-digest", () => {
|
|
45
|
-
const original =
|
|
44
|
+
const original = createTDigest();
|
|
46
45
|
const serialized = serializeTDigest(original);
|
|
47
46
|
const restored = deserializeTDigest(serialized);
|
|
48
47
|
|
|
@@ -52,7 +51,7 @@ describe("t-digest serialization", () => {
|
|
|
52
51
|
});
|
|
53
52
|
|
|
54
53
|
it("serializes and deserializes t-digest with values", () => {
|
|
55
|
-
const original =
|
|
54
|
+
const original = createTDigest();
|
|
56
55
|
original.push(100);
|
|
57
56
|
original.push(200);
|
|
58
57
|
original.push(300);
|
|
@@ -67,7 +66,7 @@ describe("t-digest serialization", () => {
|
|
|
67
66
|
});
|
|
68
67
|
|
|
69
68
|
it("preserves p95 accuracy after serialization", () => {
|
|
70
|
-
const original =
|
|
69
|
+
const original = createTDigest();
|
|
71
70
|
// Add 100 values from 1 to 100
|
|
72
71
|
for (let i = 1; i <= 100; i++) {
|
|
73
72
|
original.push(i);
|
|
@@ -85,7 +84,7 @@ describe("t-digest serialization", () => {
|
|
|
85
84
|
|
|
86
85
|
it("handles incremental updates correctly", () => {
|
|
87
86
|
// First batch
|
|
88
|
-
const digest1 =
|
|
87
|
+
const digest1 = createTDigest();
|
|
89
88
|
digest1.push(100);
|
|
90
89
|
digest1.push(110);
|
|
91
90
|
|
|
@@ -254,7 +253,7 @@ describe("incrementHourlyAggregate", () => {
|
|
|
254
253
|
it("updates min/max when existing aggregate has values", async () => {
|
|
255
254
|
// Set up existing aggregate
|
|
256
255
|
existingAggregate = {
|
|
257
|
-
tdigestState: serializeTDigest(
|
|
256
|
+
tdigestState: serializeTDigest(createTDigest()),
|
|
258
257
|
minLatencyMs: 100,
|
|
259
258
|
maxLatencyMs: 200,
|
|
260
259
|
};
|
|
@@ -309,7 +308,7 @@ describe("incrementHourlyAggregate", () => {
|
|
|
309
308
|
|
|
310
309
|
it("updates max when new latency is higher", async () => {
|
|
311
310
|
existingAggregate = {
|
|
312
|
-
tdigestState: serializeTDigest(
|
|
311
|
+
tdigestState: serializeTDigest(createTDigest()),
|
|
313
312
|
minLatencyMs: 100,
|
|
314
313
|
maxLatencyMs: 200,
|
|
315
314
|
};
|
|
@@ -361,7 +360,7 @@ describe("incrementHourlyAggregate", () => {
|
|
|
361
360
|
|
|
362
361
|
it("accumulates t-digest state across multiple runs", async () => {
|
|
363
362
|
// First run
|
|
364
|
-
const digest1 =
|
|
363
|
+
const digest1 = createTDigest();
|
|
365
364
|
digest1.push(100);
|
|
366
365
|
digest1.push(200);
|
|
367
366
|
digest1.push(300);
|
|
@@ -2,6 +2,10 @@ import type {
|
|
|
2
2
|
ScopedQueryRunner,
|
|
3
3
|
CollectorRegistry,
|
|
4
4
|
} from "@checkstack/backend-api";
|
|
5
|
+
import {
|
|
6
|
+
pushValuesIntoState,
|
|
7
|
+
percentileFromState,
|
|
8
|
+
} from "@checkstack/backend-api";
|
|
5
9
|
import {
|
|
6
10
|
ASSERTIONS_AGG_KEY,
|
|
7
11
|
AssertionOutcomeSchema,
|
|
@@ -9,7 +13,6 @@ import {
|
|
|
9
13
|
readAssertionStats,
|
|
10
14
|
type AssertionOutcome,
|
|
11
15
|
} from "@checkstack/healthcheck-common";
|
|
12
|
-
import { TDigest } from "tdigest";
|
|
13
16
|
import * as schema from "./schema";
|
|
14
17
|
import { healthCheckAggregates } from "./schema";
|
|
15
18
|
import { eq, and, sql } from "drizzle-orm";
|
|
@@ -30,46 +33,6 @@ export function getHourBucketStart(timestamp: Date): Date {
|
|
|
30
33
|
return bucketStart;
|
|
31
34
|
}
|
|
32
35
|
|
|
33
|
-
/**
|
|
34
|
-
* Serialize a t-digest to an array of numbers for storage.
|
|
35
|
-
* Format: [centroid1_mean, centroid1_n, centroid2_mean, centroid2_n, ...]
|
|
36
|
-
*/
|
|
37
|
-
export function serializeTDigest(digest: TDigest): number[] {
|
|
38
|
-
const centroids = digest.toArray();
|
|
39
|
-
const result: number[] = [];
|
|
40
|
-
for (const c of centroids) {
|
|
41
|
-
result.push(c.mean, c.n);
|
|
42
|
-
}
|
|
43
|
-
return result;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Deserialize a t-digest from storage format.
|
|
48
|
-
*/
|
|
49
|
-
export function deserializeTDigest(state: number[]): TDigest {
|
|
50
|
-
const digest = new TDigest();
|
|
51
|
-
|
|
52
|
-
if (state.length === 0) {
|
|
53
|
-
return digest;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Reconstruct centroids from pairs of [mean, n]
|
|
57
|
-
const centroids: Array<{ mean: number; n: number }> = [];
|
|
58
|
-
for (let i = 0; i < state.length; i += 2) {
|
|
59
|
-
const mean = state[i];
|
|
60
|
-
const n = state[i + 1];
|
|
61
|
-
if (mean !== undefined && n !== undefined && n > 0) {
|
|
62
|
-
centroids.push({ mean, n });
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (centroids.length > 0) {
|
|
67
|
-
digest.push_centroid(centroids);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return digest;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
36
|
interface IncrementHourlyAggregateParams {
|
|
74
37
|
db: Db;
|
|
75
38
|
systemId: string;
|
|
@@ -177,15 +140,14 @@ export async function incrementHourlyAggregate(
|
|
|
177
140
|
max = latencyMs;
|
|
178
141
|
}
|
|
179
142
|
|
|
180
|
-
// Update t-digest for p95 calculation
|
|
181
|
-
const
|
|
182
|
-
existing?.tdigestState
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
const p95 = Math.round(digest.percentile(0.95));
|
|
143
|
+
// Update t-digest for p95 calculation (shared backend-api helper).
|
|
144
|
+
const tdigestState = pushValuesIntoState({
|
|
145
|
+
state: existing?.tdigestState ?? null,
|
|
146
|
+
values: [latencyMs],
|
|
147
|
+
});
|
|
148
|
+
const p95 = Math.round(
|
|
149
|
+
percentileFromState({ state: tdigestState, q: 0.95 }) ?? latencyMs,
|
|
150
|
+
);
|
|
189
151
|
|
|
190
152
|
return { latencySumIncr, min, max, tdigestState, p95 };
|
|
191
153
|
})();
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, it, expect, mock } from "bun:test";
|
|
2
|
+
import { call } from "@orpc/server";
|
|
3
|
+
import { createMockRpcContext } from "@checkstack/backend-api";
|
|
4
|
+
import { createHealthCheckRouter } from "./router";
|
|
5
|
+
import { createStubHealthCheckCache } from "./cache-test-stub";
|
|
6
|
+
import type { HealthRunReaction } from "./queue-executor";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A satellite result MUST drive the SAME core post-run path a local run does:
|
|
10
|
+
* the reactive `health` entity write, the notification, the automation hooks,
|
|
11
|
+
* etc. The router routes it there via the injected `reactToSatelliteRun`
|
|
12
|
+
* reactor. These tests pin that wiring so ingest can never silently regress to
|
|
13
|
+
* an insert-only path (which is what left satellite-detected outages silent).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
// A service principal - ingestSatelliteResult is a backend-to-backend proc.
|
|
17
|
+
const serviceContext = () =>
|
|
18
|
+
createMockRpcContext({
|
|
19
|
+
user: { type: "service", pluginId: "satellite-backend" } as never,
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
function buildRouter(opts: {
|
|
23
|
+
reactToSatelliteRun?: (run: HealthRunReaction) => Promise<void>;
|
|
24
|
+
transaction?: ReturnType<typeof mock>;
|
|
25
|
+
}) {
|
|
26
|
+
// db.select(...).from(...).where(...) resolves the config row (name +
|
|
27
|
+
// collectors) that processSatelliteResult reads.
|
|
28
|
+
const database = {
|
|
29
|
+
select: mock(() => ({
|
|
30
|
+
from: mock(() => ({
|
|
31
|
+
where: mock(() =>
|
|
32
|
+
Promise.resolve([{ name: "Ping", collectors: [] }]),
|
|
33
|
+
),
|
|
34
|
+
})),
|
|
35
|
+
})),
|
|
36
|
+
transaction:
|
|
37
|
+
opts.transaction ??
|
|
38
|
+
mock(async () => {
|
|
39
|
+
/* not expected in the reactor path */
|
|
40
|
+
}),
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const collectorRegistry = {
|
|
44
|
+
register: mock(() => {}),
|
|
45
|
+
getCollector: mock(() => undefined),
|
|
46
|
+
getCollectors: mock(() => []),
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const catalogClient = {
|
|
50
|
+
getSystem: mock(async () => ({ id: "sys-1", name: "API Server" })),
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
return createHealthCheckRouter({
|
|
54
|
+
database: database as never,
|
|
55
|
+
registry: { getStrategy: mock(() => undefined) } as never,
|
|
56
|
+
collectorRegistry: collectorRegistry as never,
|
|
57
|
+
gitOpsClient: {} as never,
|
|
58
|
+
getEmitHook: () => undefined,
|
|
59
|
+
cache: createStubHealthCheckCache(),
|
|
60
|
+
configService: {} as never,
|
|
61
|
+
catalogClient: catalogClient as never,
|
|
62
|
+
maintenanceClient: {} as never,
|
|
63
|
+
logger: {
|
|
64
|
+
debug: mock(() => {}),
|
|
65
|
+
info: mock(() => {}),
|
|
66
|
+
warn: mock(() => {}),
|
|
67
|
+
error: mock(() => {}),
|
|
68
|
+
} as never,
|
|
69
|
+
...(opts.reactToSatelliteRun
|
|
70
|
+
? { reactToSatelliteRun: opts.reactToSatelliteRun }
|
|
71
|
+
: {}),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const ingestInput = {
|
|
76
|
+
configId: "config-1",
|
|
77
|
+
systemId: "sys-1",
|
|
78
|
+
status: "unhealthy" as const,
|
|
79
|
+
latencyMs: 123,
|
|
80
|
+
result: {
|
|
81
|
+
status: "unhealthy" as const,
|
|
82
|
+
latencyMs: 123,
|
|
83
|
+
message: "Check failed",
|
|
84
|
+
metadata: { connected: true, collectors: {} },
|
|
85
|
+
},
|
|
86
|
+
executedAt: "2026-07-03T10:00:00.000Z",
|
|
87
|
+
sourceId: "sat-eu",
|
|
88
|
+
sourceLabel: "EU West",
|
|
89
|
+
environmentId: "env-prod",
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
describe("ingestSatelliteResult - drives the shared post-run path", () => {
|
|
93
|
+
it("routes a satellite result through reactToSatelliteRun with its processed payload", async () => {
|
|
94
|
+
const runs: HealthRunReaction[] = [];
|
|
95
|
+
const router = buildRouter({
|
|
96
|
+
reactToSatelliteRun: async (run) => {
|
|
97
|
+
runs.push(run);
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
await call(router.ingestSatelliteResult, ingestInput, {
|
|
102
|
+
context: serviceContext(),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
expect(runs).toHaveLength(1);
|
|
106
|
+
const run = runs[0];
|
|
107
|
+
// The run carries the satellite's SOURCE + environment, and its display
|
|
108
|
+
// name was resolved for the notification the shared path will send.
|
|
109
|
+
expect(run.systemId).toBe("sys-1");
|
|
110
|
+
expect(run.systemName).toBe("API Server");
|
|
111
|
+
expect(run.configId).toBe("config-1");
|
|
112
|
+
expect(run.configName).toBe("Ping");
|
|
113
|
+
expect(run.environmentId).toBe("env-prod");
|
|
114
|
+
expect(run.status).toBe("unhealthy");
|
|
115
|
+
expect(run.sourceId).toBe("sat-eu");
|
|
116
|
+
expect(run.sourceLabel).toBe("EU West");
|
|
117
|
+
expect(run.runTimestamp).toEqual(new Date("2026-07-03T10:00:00.000Z"));
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("falls back to an insert-only persist when no reactor is wired", async () => {
|
|
121
|
+
// Record that the insert-only fallback opened a transaction, without
|
|
122
|
+
// executing its body (the aggregate internals need a full db mock and are
|
|
123
|
+
// covered elsewhere); the point is that ingest still records the run.
|
|
124
|
+
const transaction = mock(async () => {});
|
|
125
|
+
const router = buildRouter({ transaction });
|
|
126
|
+
|
|
127
|
+
await call(router.ingestSatelliteResult, ingestInput, {
|
|
128
|
+
context: serviceContext(),
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
// The insert-only fallback ran (a run is still recorded), but there is no
|
|
132
|
+
// reactor path to drive notifications/automations - which is exactly why
|
|
133
|
+
// the real host always wires `reactToSatelliteRun`.
|
|
134
|
+
expect(transaction).toHaveBeenCalledTimes(1);
|
|
135
|
+
});
|
|
136
|
+
});
|
package/src/router.ts
CHANGED
|
@@ -46,6 +46,7 @@ import { CatalogApi } from "@checkstack/catalog-common";
|
|
|
46
46
|
import { MaintenanceApi } from "@checkstack/maintenance-common";
|
|
47
47
|
import type { Logger } from "@checkstack/backend-api";
|
|
48
48
|
import type { HealthCheckCache } from "./cache";
|
|
49
|
+
import type { HealthRunReaction } from "./queue-executor";
|
|
49
50
|
import {
|
|
50
51
|
applySystemHealthOverrides,
|
|
51
52
|
type SystemHealthOverrideReader,
|
|
@@ -104,6 +105,17 @@ export const createHealthCheckRouter = (opts: {
|
|
|
104
105
|
* Optional so tests / no-incident deployments simply skip the fold.
|
|
105
106
|
*/
|
|
106
107
|
incidentHealthOverrideReader?: SystemHealthOverrideReader;
|
|
108
|
+
/**
|
|
109
|
+
* Drives the SHARED core post-run path for an ingested satellite result -
|
|
110
|
+
* the reactive `health` entity write, cache reconcile, realtime signal,
|
|
111
|
+
* automation hooks, transition record, and subscriber notification - so a
|
|
112
|
+
* satellite-detected change reacts exactly like a local run. The host binds
|
|
113
|
+
* the service dependencies once (via a `persistRunAndReact` closure) and
|
|
114
|
+
* passes this narrowed reactor, so the local and satellite callers cannot
|
|
115
|
+
* pass different deps and drift. Optional so tests may omit it, in which case
|
|
116
|
+
* ingest falls back to an insert-only persist.
|
|
117
|
+
*/
|
|
118
|
+
reactToSatelliteRun?: (run: HealthRunReaction) => Promise<void>;
|
|
107
119
|
}) => {
|
|
108
120
|
const {
|
|
109
121
|
database,
|
|
@@ -118,6 +130,7 @@ export const createHealthCheckRouter = (opts: {
|
|
|
118
130
|
signalService,
|
|
119
131
|
recomputeSystemRollupHealth,
|
|
120
132
|
incidentHealthOverrideReader,
|
|
133
|
+
reactToSatelliteRun,
|
|
121
134
|
} = opts;
|
|
122
135
|
// Create service instance once - shared across all handlers
|
|
123
136
|
const service = new HealthCheckService(
|
|
@@ -612,6 +625,7 @@ export const createHealthCheckRouter = (opts: {
|
|
|
612
625
|
enabled: input.body.enabled,
|
|
613
626
|
stateThresholds: input.body.stateThresholds,
|
|
614
627
|
satelliteIds: input.body.satelliteIds,
|
|
628
|
+
satelliteEnvironmentIds: input.body.satelliteEnvironmentIds,
|
|
615
629
|
environmentIds: input.body.environmentIds,
|
|
616
630
|
includeLocal: input.body.includeLocal,
|
|
617
631
|
notificationPolicy: input.body.notificationPolicy,
|
|
@@ -634,6 +648,7 @@ export const createHealthCheckRouter = (opts: {
|
|
|
634
648
|
enabled: input.enabled,
|
|
635
649
|
stateThresholds: input.stateThresholds,
|
|
636
650
|
satelliteIds: input.satelliteIds,
|
|
651
|
+
satelliteEnvironmentIds: input.satelliteEnvironmentIds,
|
|
637
652
|
environmentIds: input.environmentIds,
|
|
638
653
|
includeLocal: input.includeLocal,
|
|
639
654
|
notificationPolicy: input.notificationPolicy,
|
|
@@ -899,10 +914,56 @@ export const createHealthCheckRouter = (opts: {
|
|
|
899
914
|
|
|
900
915
|
ingestSatelliteResult: os.ingestSatelliteResult.handler(
|
|
901
916
|
async ({ input }) => {
|
|
902
|
-
|
|
903
|
-
//
|
|
904
|
-
|
|
905
|
-
|
|
917
|
+
// Process the satellite's raw result (evaluate assertions, strip
|
|
918
|
+
// ephemeral fields, resolve the check name) WITHOUT persisting...
|
|
919
|
+
const { status, resultRecord, configName } =
|
|
920
|
+
await service.processSatelliteResult({
|
|
921
|
+
configId: input.configId,
|
|
922
|
+
status: input.status,
|
|
923
|
+
result: input.result,
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
if (reactToSatelliteRun) {
|
|
927
|
+
// ...then persist + REACT through the SAME core post-run path a local
|
|
928
|
+
// run uses (reactive entity, cache reconcile, signals, automation
|
|
929
|
+
// hooks, transition, notification). Previously ingest only inserted
|
|
930
|
+
// the row, so a satellite-detected outage fired no notifications or
|
|
931
|
+
// automations - routing it here is what keeps the two from drifting.
|
|
932
|
+
const system = await catalogClient
|
|
933
|
+
.getSystem({ systemId: input.systemId })
|
|
934
|
+
.catch(() => null);
|
|
935
|
+
await reactToSatelliteRun({
|
|
936
|
+
systemId: input.systemId,
|
|
937
|
+
systemName: system?.name ?? input.systemId,
|
|
938
|
+
configId: input.configId,
|
|
939
|
+
configName,
|
|
940
|
+
environmentId: input.environmentId ?? null,
|
|
941
|
+
// The env NAME is intentionally omitted (no extra catalog
|
|
942
|
+
// round-trip): the notifier falls back to the system name.
|
|
943
|
+
status,
|
|
944
|
+
latencyMs: input.latencyMs,
|
|
945
|
+
result: resultRecord,
|
|
946
|
+
sourceId: input.sourceId,
|
|
947
|
+
sourceLabel: input.sourceLabel,
|
|
948
|
+
runTimestamp: new Date(input.executedAt),
|
|
949
|
+
});
|
|
950
|
+
} else {
|
|
951
|
+
// No reactor wired (older host / test harness): fall back to a bare
|
|
952
|
+
// persist + cache refresh so the run is still recorded. The real host
|
|
953
|
+
// always wires the reactor above.
|
|
954
|
+
await service.insertSatelliteRun({
|
|
955
|
+
configId: input.configId,
|
|
956
|
+
systemId: input.systemId,
|
|
957
|
+
environmentId: input.environmentId ?? null,
|
|
958
|
+
status,
|
|
959
|
+
latencyMs: input.latencyMs,
|
|
960
|
+
result: resultRecord,
|
|
961
|
+
sourceId: input.sourceId,
|
|
962
|
+
sourceLabel: input.sourceLabel,
|
|
963
|
+
executedAt: input.executedAt,
|
|
964
|
+
});
|
|
965
|
+
await cache.invalidateSystem(input.systemId);
|
|
966
|
+
}
|
|
906
967
|
},
|
|
907
968
|
),
|
|
908
969
|
|
package/src/schema.ts
CHANGED
|
@@ -111,6 +111,27 @@ export const systemHealthChecks = pgTable(
|
|
|
111
111
|
* `length === 0`; jsonb stores both faithfully.
|
|
112
112
|
*/
|
|
113
113
|
environmentIds: jsonb("environment_ids").$type<string[]>(),
|
|
114
|
+
/**
|
|
115
|
+
* Per-SATELLITE environment scoping, keyed by satellite id.
|
|
116
|
+
*
|
|
117
|
+
* Additive on top of `satelliteIds`, which still answers "which satellites
|
|
118
|
+
* run this". This answers "and for which environments", so a prod satellite
|
|
119
|
+
* can run only the prod environment instead of probing every environment
|
|
120
|
+
* from a network it may have no route to.
|
|
121
|
+
*
|
|
122
|
+
* Semantics per entry, mirroring `environmentIds`:
|
|
123
|
+
* - key ABSENT (or `null`) => that satellite runs every environment the
|
|
124
|
+
* assignment resolves to. This is the backfilled default, so existing
|
|
125
|
+
* rows keep behaving as they did.
|
|
126
|
+
* - `[]` => opt out: that satellite runs ONCE with no environment.
|
|
127
|
+
* - non-empty => exactly those ids, INTERSECTED with the assignment's own
|
|
128
|
+
* effective set - a satellite can never widen the assignment's scope,
|
|
129
|
+
* only narrow it.
|
|
130
|
+
*/
|
|
131
|
+
satelliteEnvironmentIds:
|
|
132
|
+
jsonb("satellite_environment_ids").$type<
|
|
133
|
+
Record<string, string[] | null>
|
|
134
|
+
>(),
|
|
114
135
|
/**
|
|
115
136
|
* Whether to also run this check locally on the core instance.
|
|
116
137
|
* Defaults to true. Only relevant when satelliteIds is set.
|
|
@@ -97,7 +97,9 @@ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
|
|
|
97
97
|
|
|
98
98
|
const result = await service.getSystemHealthStatus("system-1");
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
// `unknown`, not `healthy`: a system with no checks is unmeasured, and this
|
|
101
|
+
// test is about transaction batching, not about inventing a status.
|
|
102
|
+
expect(result.status).toBe("unknown");
|
|
101
103
|
expect(result.checkStatuses).toHaveLength(0);
|
|
102
104
|
const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
|
|
103
105
|
.transaction;
|
|
@@ -13,8 +13,9 @@ import { HealthCheckService } from "./service";
|
|
|
13
13
|
* Satellite ingest evaluates assertions ON THE CORE (satellites never held
|
|
14
14
|
* the assertion semantics — before this, satellite-executed checks silently
|
|
15
15
|
* skipped assertions), then strips ephemeral fields for parity with local
|
|
16
|
-
* runs.
|
|
17
|
-
*
|
|
16
|
+
* runs. `processSatelliteResult` returns the processed status + result record
|
|
17
|
+
* (and the check name); the shared post-run path persists it. These tests
|
|
18
|
+
* assert on that returned payload.
|
|
18
19
|
*/
|
|
19
20
|
|
|
20
21
|
const KEY = computeAssertionKey({
|
|
@@ -30,43 +31,17 @@ const collectorResultSchema = z.object({
|
|
|
30
31
|
body: healthResultString({ "x-ephemeral": true }),
|
|
31
32
|
});
|
|
32
33
|
|
|
33
|
-
function buildService({
|
|
34
|
-
|
|
35
|
-
inserted,
|
|
36
|
-
}: {
|
|
37
|
-
entries: CollectorConfigEntry[];
|
|
38
|
-
inserted: Record<string, unknown>[];
|
|
39
|
-
}) {
|
|
40
|
-
const tx = {
|
|
41
|
-
insert: mock(() => ({
|
|
42
|
-
values: mock((vals: Record<string, unknown>) => {
|
|
43
|
-
inserted.push(vals);
|
|
44
|
-
return Object.assign(Promise.resolve(), {
|
|
45
|
-
onConflictDoUpdate: mock(() => Promise.resolve()),
|
|
46
|
-
onConflictDoNothing: mock(() => Promise.resolve()),
|
|
47
|
-
});
|
|
48
|
-
}),
|
|
49
|
-
})),
|
|
34
|
+
function buildService({ entries }: { entries: CollectorConfigEntry[] }) {
|
|
35
|
+
const db = {
|
|
50
36
|
select: mock(() => ({
|
|
51
37
|
from: mock(() => ({
|
|
52
38
|
where: mock(() =>
|
|
53
|
-
|
|
54
|
-
limit: mock(() => Promise.resolve([])),
|
|
55
|
-
}),
|
|
39
|
+
Promise.resolve([{ name: "Test check", collectors: entries }]),
|
|
56
40
|
),
|
|
57
41
|
})),
|
|
58
42
|
})),
|
|
59
43
|
};
|
|
60
44
|
|
|
61
|
-
const db = {
|
|
62
|
-
select: mock(() => ({
|
|
63
|
-
from: mock(() => ({
|
|
64
|
-
where: mock(() => Promise.resolve([{ collectors: entries }])),
|
|
65
|
-
})),
|
|
66
|
-
})),
|
|
67
|
-
transaction: mock(async (fn: (t: typeof tx) => Promise<void>) => fn(tx)),
|
|
68
|
-
};
|
|
69
|
-
|
|
70
45
|
const collectorRegistry = {
|
|
71
46
|
register: mock(() => {}),
|
|
72
47
|
getCollector: mock(() => ({
|
|
@@ -113,64 +88,59 @@ const entries: CollectorConfigEntry[] = [
|
|
|
113
88
|
},
|
|
114
89
|
];
|
|
115
90
|
|
|
116
|
-
async function
|
|
91
|
+
async function process({
|
|
117
92
|
entries,
|
|
118
93
|
statusCode,
|
|
119
94
|
}: {
|
|
120
95
|
entries: CollectorConfigEntry[];
|
|
121
96
|
statusCode: number;
|
|
122
97
|
}) {
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
await service.ingestSatelliteResult({
|
|
98
|
+
const service = buildService({ entries });
|
|
99
|
+
return service.processSatelliteResult({
|
|
126
100
|
configId: "config-1",
|
|
127
|
-
systemId: "system-1",
|
|
128
101
|
status: "healthy",
|
|
129
|
-
latencyMs: 42,
|
|
130
102
|
result: satelliteResult({ statusCode }) as never,
|
|
131
|
-
executedAt: "2026-07-03T10:00:00.000Z",
|
|
132
|
-
sourceId: "sat-1",
|
|
133
|
-
sourceLabel: "EU West",
|
|
134
103
|
});
|
|
135
|
-
const runInsert = inserted.find((v) => "status" in v && "result" in v);
|
|
136
|
-
expect(runInsert).toBeDefined();
|
|
137
|
-
return runInsert as Record<string, unknown>;
|
|
138
104
|
}
|
|
139
105
|
|
|
140
|
-
function collectorEntryOf(
|
|
141
|
-
const result =
|
|
106
|
+
function collectorEntryOf(resultRecord: Record<string, unknown>) {
|
|
107
|
+
const result = resultRecord as {
|
|
142
108
|
metadata: { collectors: Record<string, Record<string, unknown>> };
|
|
143
109
|
};
|
|
144
110
|
return result.metadata.collectors["entry-1"];
|
|
145
111
|
}
|
|
146
112
|
|
|
147
|
-
describe("
|
|
113
|
+
describe("processSatelliteResult - assertion evaluation at ingest", () => {
|
|
148
114
|
it("downgrades a satellite-healthy run whose assertion fails", async () => {
|
|
149
|
-
const
|
|
150
|
-
expect(
|
|
115
|
+
const { status, resultRecord } = await process({ entries, statusCode: 404 });
|
|
116
|
+
expect(status).toBe("unhealthy");
|
|
151
117
|
|
|
152
|
-
const entry = collectorEntryOf(
|
|
118
|
+
const entry = collectorEntryOf(resultRecord);
|
|
153
119
|
expect(entry._assertionFailed).toBe("statusCode equals 200");
|
|
154
120
|
expect(entry._assertions).toEqual([
|
|
155
121
|
expect.objectContaining({ key: KEY, passed: false, actual: "404" }),
|
|
156
122
|
]);
|
|
157
|
-
|
|
158
|
-
expect(message).toBe(
|
|
123
|
+
expect((resultRecord as { message: string }).message).toBe(
|
|
159
124
|
"Check failed: Assertion failed: statusCode equals 200",
|
|
160
125
|
);
|
|
161
126
|
});
|
|
162
127
|
|
|
163
128
|
it("keeps a passing run healthy and stores the passing outcome", async () => {
|
|
164
|
-
const
|
|
165
|
-
expect(
|
|
129
|
+
const { status, resultRecord } = await process({ entries, statusCode: 200 });
|
|
130
|
+
expect(status).toBe("healthy");
|
|
166
131
|
|
|
167
|
-
const entry = collectorEntryOf(
|
|
132
|
+
const entry = collectorEntryOf(resultRecord);
|
|
168
133
|
expect(entry._assertionFailed).toBeUndefined();
|
|
169
134
|
expect(entry._assertions).toEqual([
|
|
170
135
|
expect.objectContaining({ key: KEY, passed: true, actual: "200" }),
|
|
171
136
|
]);
|
|
172
137
|
});
|
|
173
138
|
|
|
139
|
+
it("resolves the check name for the notification", async () => {
|
|
140
|
+
const { configName } = await process({ entries, statusCode: 200 });
|
|
141
|
+
expect(configName).toBe("Test check");
|
|
142
|
+
});
|
|
143
|
+
|
|
174
144
|
it("strips ephemeral fields AFTER assertions ran against them", async () => {
|
|
175
145
|
const withBodyAssertion: CollectorConfigEntry[] = [
|
|
176
146
|
{
|
|
@@ -187,12 +157,12 @@ describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
|
|
|
187
157
|
],
|
|
188
158
|
},
|
|
189
159
|
];
|
|
190
|
-
const
|
|
160
|
+
const { status, resultRecord } = await process({
|
|
191
161
|
entries: withBodyAssertion,
|
|
192
162
|
statusCode: 200,
|
|
193
163
|
});
|
|
194
164
|
|
|
195
|
-
const entry = collectorEntryOf(
|
|
165
|
+
const entry = collectorEntryOf(resultRecord);
|
|
196
166
|
// The JSONPath assertion evaluated against the (ephemeral) body...
|
|
197
167
|
expect(entry._assertions).toEqual([
|
|
198
168
|
expect.objectContaining({ passed: true, actual: "ok" }),
|
|
@@ -200,14 +170,17 @@ describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
|
|
|
200
170
|
// ...but the body itself never reaches storage.
|
|
201
171
|
expect(entry.body).toBeUndefined();
|
|
202
172
|
expect(entry.statusCode).toBe(200);
|
|
203
|
-
expect(
|
|
173
|
+
expect(status).toBe("healthy");
|
|
204
174
|
});
|
|
205
175
|
|
|
206
176
|
it("tolerates collector entries the config no longer knows", async () => {
|
|
207
|
-
const
|
|
177
|
+
const { status, resultRecord } = await process({
|
|
178
|
+
entries: [],
|
|
179
|
+
statusCode: 500,
|
|
180
|
+
});
|
|
208
181
|
// No assertions configured: status passes through untouched.
|
|
209
|
-
expect(
|
|
210
|
-
const entry = collectorEntryOf(
|
|
182
|
+
expect(status).toBe("healthy");
|
|
183
|
+
const entry = collectorEntryOf(resultRecord);
|
|
211
184
|
expect(entry._assertions).toBeUndefined();
|
|
212
185
|
});
|
|
213
186
|
});
|
|
@@ -159,9 +159,12 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
159
159
|
const result = await service.getSystemHealthStatus("system-1");
|
|
160
160
|
|
|
161
161
|
// The post-filter associations list is empty, so the system has no
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
|
|
162
|
+
// ACTIVE checks. Paused failures still do NOT keep the system degraded -
|
|
163
|
+
// that is the behaviour this test guards - but the result is now
|
|
164
|
+
// `unknown` rather than `healthy`: with its only check paused, nothing is
|
|
165
|
+
// measuring this system, and claiming health it has no evidence for is
|
|
166
|
+
// what made a broken check read green on the catalog and status page.
|
|
167
|
+
expect(result.status).toBe("unknown");
|
|
165
168
|
expect(result.checkStatuses).toHaveLength(0);
|
|
166
169
|
});
|
|
167
170
|
|
|
@@ -260,7 +263,9 @@ describe("HealthCheckService - paused configuration filtering", () => {
|
|
|
260
263
|
|
|
261
264
|
const result = await service.getSystemHealthStatus("system-1");
|
|
262
265
|
|
|
263
|
-
|
|
266
|
+
// No enabled associations = nothing measured, so `unknown` rather than an
|
|
267
|
+
// invented `healthy`.
|
|
268
|
+
expect(result.status).toBe("unknown");
|
|
264
269
|
expect(result.checkStatuses).toHaveLength(0);
|
|
265
270
|
});
|
|
266
271
|
});
|