@checkstack/healthcheck-backend 1.21.3 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +314 -0
- package/drizzle/0021_amazing_wolf_cub.sql +8 -0
- package/drizzle/meta/0021_snapshot.json +717 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +31 -29
- package/src/ai/system-signals-contributor.test.ts +1 -0
- package/src/cache.test.ts +3 -0
- package/src/effective-environments.test.ts +63 -2
- package/src/effective-environments.ts +34 -0
- package/src/health-entity.ts +8 -2
- package/src/health-state.ts +15 -6
- package/src/index.ts +33 -0
- package/src/queue-executor.test.ts +270 -0
- package/src/queue-executor.ts +672 -569
- package/src/router-satellite-ingest.test.ts +136 -0
- package/src/router.ts +65 -4
- package/src/satellite-liveness.test.ts +199 -0
- package/src/satellite-liveness.ts +106 -0
- package/src/schema.ts +21 -0
- package/src/service-batching.test.ts +3 -1
- package/src/service-ingest-assertions.test.ts +33 -60
- package/src/service-paused-filter.test.ts +9 -4
- package/src/service-rollup-worst-wins.test.ts +147 -55
- package/src/service.ts +355 -148
- package/src/state-evaluator.test.ts +49 -0
- package/src/system-health-override.ts +6 -1
- package/tsconfig.json +6 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/healthcheck-backend",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.23.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -14,32 +14,34 @@
|
|
|
14
14
|
"lint:code": "eslint . --max-warnings 0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@checkstack/ai-backend": "0.11.
|
|
18
|
-
"@checkstack/ai-common": "0.6.
|
|
19
|
-
"@checkstack/automation-backend": "0.11.
|
|
20
|
-
"@checkstack/backend-api": "0.
|
|
21
|
-
"@checkstack/cache-api": "0.3.
|
|
22
|
-
"@checkstack/cache-utils": "0.3.
|
|
23
|
-
"@checkstack/catalog-backend": "1.10.
|
|
24
|
-
"@checkstack/catalog-common": "2.8.
|
|
25
|
-
"@checkstack/command-backend": "0.
|
|
26
|
-
"@checkstack/common": "0.
|
|
27
|
-
"@checkstack/gitops-backend": "0.5.
|
|
28
|
-
"@checkstack/gitops-common": "0.7.
|
|
29
|
-
"@checkstack/healthcheck-common": "1.
|
|
30
|
-
"@checkstack/
|
|
31
|
-
"@checkstack/incident-
|
|
32
|
-
"@checkstack/
|
|
33
|
-
"@checkstack/
|
|
34
|
-
"@checkstack/
|
|
35
|
-
"@checkstack/
|
|
36
|
-
"@checkstack/
|
|
37
|
-
"@checkstack/
|
|
38
|
-
"@checkstack/
|
|
39
|
-
"@checkstack/
|
|
40
|
-
"@checkstack/
|
|
41
|
-
"@checkstack/
|
|
42
|
-
"@checkstack/
|
|
17
|
+
"@checkstack/ai-backend": "0.11.5",
|
|
18
|
+
"@checkstack/ai-common": "0.6.8",
|
|
19
|
+
"@checkstack/automation-backend": "0.11.9",
|
|
20
|
+
"@checkstack/backend-api": "0.35.0",
|
|
21
|
+
"@checkstack/cache-api": "0.3.21",
|
|
22
|
+
"@checkstack/cache-utils": "0.3.2",
|
|
23
|
+
"@checkstack/catalog-backend": "1.10.2",
|
|
24
|
+
"@checkstack/catalog-common": "2.8.2",
|
|
25
|
+
"@checkstack/command-backend": "0.3.0",
|
|
26
|
+
"@checkstack/common": "0.24.0",
|
|
27
|
+
"@checkstack/gitops-backend": "0.5.28",
|
|
28
|
+
"@checkstack/gitops-common": "0.7.5",
|
|
29
|
+
"@checkstack/healthcheck-common": "1.19.1",
|
|
30
|
+
"@checkstack/healthcheck-execution": "0.35.1",
|
|
31
|
+
"@checkstack/incident-backend": "1.14.0",
|
|
32
|
+
"@checkstack/incident-common": "1.11.0",
|
|
33
|
+
"@checkstack/maintenance-common": "1.11.0",
|
|
34
|
+
"@checkstack/notification-common": "1.9.0",
|
|
35
|
+
"@checkstack/queue-api": "0.4.1",
|
|
36
|
+
"@checkstack/satellite-backend": "0.10.0",
|
|
37
|
+
"@checkstack/satellite-common": "0.12.0",
|
|
38
|
+
"@checkstack/script-packages-backend": "0.4.7",
|
|
39
|
+
"@checkstack/sdk": "0.136.1",
|
|
40
|
+
"@checkstack/secrets-backend": "0.3.10",
|
|
41
|
+
"@checkstack/secrets-common": "0.3.4",
|
|
42
|
+
"@checkstack/signal-common": "0.3.2",
|
|
43
|
+
"@checkstack/status-page-backend": "0.7.0",
|
|
44
|
+
"@checkstack/status-page-common": "0.7.0",
|
|
43
45
|
"@hono/zod-validator": "^0.7.6",
|
|
44
46
|
"@orpc/contract": "^1.14.4",
|
|
45
47
|
"@orpc/server": "^1.14.4",
|
|
@@ -51,8 +53,8 @@
|
|
|
51
53
|
},
|
|
52
54
|
"devDependencies": {
|
|
53
55
|
"@checkstack/drizzle-helper": "0.0.6",
|
|
54
|
-
"@checkstack/scripts": "0.7.
|
|
55
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
56
|
+
"@checkstack/scripts": "0.7.7",
|
|
57
|
+
"@checkstack/test-utils-backend": "0.1.62",
|
|
56
58
|
"@checkstack/tsconfig": "0.0.7",
|
|
57
59
|
"@types/bun": "^1.0.0",
|
|
58
60
|
"@types/pg": "^8.20.0",
|
package/src/cache.test.ts
CHANGED
|
@@ -63,6 +63,9 @@ function response(
|
|
|
63
63
|
runsConsidered: 1,
|
|
64
64
|
sliceCount: c.sliceCount ?? 1,
|
|
65
65
|
failingSliceCount: c.failingSliceCount ?? 0,
|
|
66
|
+
// These tests exercise the cache's fingerprint/invalidation behaviour,
|
|
67
|
+
// which reads the counts rather than the per-slice breakdown.
|
|
68
|
+
slices: [],
|
|
66
69
|
})),
|
|
67
70
|
};
|
|
68
71
|
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import { describe, it, expect } from "bun:test";
|
|
2
|
-
import {
|
|
1
|
+
import { describe, it, test, expect } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
resolveEffectiveEnvironments,
|
|
4
|
+
resolveSatelliteEnvironments,
|
|
5
|
+
} from "./effective-environments";
|
|
3
6
|
import type { Environment } from "@checkstack/catalog-common";
|
|
4
7
|
|
|
5
8
|
const env = (
|
|
@@ -91,3 +94,61 @@ describe("resolveEffectiveEnvironments", () => {
|
|
|
91
94
|
expect(result).toEqual([]);
|
|
92
95
|
});
|
|
93
96
|
});
|
|
97
|
+
|
|
98
|
+
describe("resolveSatelliteEnvironments", () => {
|
|
99
|
+
const effective = [
|
|
100
|
+
{ id: "env-prod", name: "Production", fields: {} },
|
|
101
|
+
{ id: "env-stage", name: "Staging", fields: {} },
|
|
102
|
+
];
|
|
103
|
+
|
|
104
|
+
test("an unscoped satellite runs every environment the assignment resolved to", () => {
|
|
105
|
+
// The backfill-free default: a NULL column means no scoping, so existing
|
|
106
|
+
// assignments keep behaving exactly as they did.
|
|
107
|
+
expect(
|
|
108
|
+
resolveSatelliteEnvironments({
|
|
109
|
+
effective,
|
|
110
|
+
satelliteEnvironmentIds: undefined,
|
|
111
|
+
}),
|
|
112
|
+
).toEqual(effective);
|
|
113
|
+
expect(
|
|
114
|
+
resolveSatelliteEnvironments({ effective, satelliteEnvironmentIds: null }),
|
|
115
|
+
).toEqual(effective);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("a scoped satellite runs only its own environments", () => {
|
|
119
|
+
// The point of the feature: the prod satellite probes prod, and never
|
|
120
|
+
// reaches for a staging endpoint it may have no route to.
|
|
121
|
+
expect(
|
|
122
|
+
resolveSatelliteEnvironments({
|
|
123
|
+
effective,
|
|
124
|
+
satelliteEnvironmentIds: ["env-prod"],
|
|
125
|
+
}).map((e) => e.id),
|
|
126
|
+
).toEqual(["env-prod"]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a satellite can NARROW but never widen the assignment's scope", () => {
|
|
130
|
+
// An id the assignment does not cover silently drops - a satellite must not
|
|
131
|
+
// be able to probe an environment the assignment itself excluded.
|
|
132
|
+
expect(
|
|
133
|
+
resolveSatelliteEnvironments({
|
|
134
|
+
effective,
|
|
135
|
+
satelliteEnvironmentIds: ["env-prod", "env-secret"],
|
|
136
|
+
}).map((e) => e.id),
|
|
137
|
+
).toEqual(["env-prod"]);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("an empty selector opts the satellite out into a single env-less run", () => {
|
|
141
|
+
expect(
|
|
142
|
+
resolveSatelliteEnvironments({ effective, satelliteEnvironmentIds: [] }),
|
|
143
|
+
).toEqual([]);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("preserves the assignment's order, so fan-out stays deterministic", () => {
|
|
147
|
+
expect(
|
|
148
|
+
resolveSatelliteEnvironments({
|
|
149
|
+
effective,
|
|
150
|
+
satelliteEnvironmentIds: ["env-stage", "env-prod"],
|
|
151
|
+
}).map((e) => e.id),
|
|
152
|
+
).toEqual(["env-prod", "env-stage"]);
|
|
153
|
+
});
|
|
154
|
+
});
|
|
@@ -62,3 +62,37 @@ export function resolveEffectiveEnvironments({
|
|
|
62
62
|
.filter((env) => wanted.has(env.id))
|
|
63
63
|
.map((env) => toEffective(env));
|
|
64
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Narrow an assignment's effective environments to those a SPECIFIC satellite
|
|
68
|
+
* should run.
|
|
69
|
+
*
|
|
70
|
+
* A satellite's selector can only ever narrow, never widen: the assignment
|
|
71
|
+
* decides which environments exist for this check, and the satellite decides
|
|
72
|
+
* which of those it is responsible for. That ordering is what lets a prod
|
|
73
|
+
* satellite run only prod without being able to reach environments the
|
|
74
|
+
* assignment itself excluded.
|
|
75
|
+
*
|
|
76
|
+
* - `undefined` / `null` => every environment the assignment resolved to. This
|
|
77
|
+
* is the default for an unscoped satellite, so existing assignments keep
|
|
78
|
+
* behaving exactly as they did.
|
|
79
|
+
* - `[]` => opt out: one env-less run on this satellite.
|
|
80
|
+
* - non-empty => exactly those ids, intersected with `effective`;
|
|
81
|
+
* an id the assignment does not cover silently drops.
|
|
82
|
+
*
|
|
83
|
+
* Order follows `effective`, so fan-out order stays deterministic.
|
|
84
|
+
*/
|
|
85
|
+
export function resolveSatelliteEnvironments({
|
|
86
|
+
effective,
|
|
87
|
+
satelliteEnvironmentIds,
|
|
88
|
+
}: {
|
|
89
|
+
effective: EffectiveEnvironment[];
|
|
90
|
+
satelliteEnvironmentIds: string[] | null | undefined;
|
|
91
|
+
}): EffectiveEnvironment[] {
|
|
92
|
+
if (satelliteEnvironmentIds === null || satelliteEnvironmentIds === undefined) {
|
|
93
|
+
return effective;
|
|
94
|
+
}
|
|
95
|
+
if (satelliteEnvironmentIds.length === 0) return [];
|
|
96
|
+
const wanted = new Set(satelliteEnvironmentIds);
|
|
97
|
+
return effective.filter((env) => wanted.has(env.id));
|
|
98
|
+
}
|
package/src/health-entity.ts
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* - the `writeHealthEntity` helper called at every evaluation-write site.
|
|
23
23
|
*/
|
|
24
24
|
import { z } from "zod";
|
|
25
|
-
import {
|
|
25
|
+
import { SystemHealthStatusSchema } from "@checkstack/healthcheck-common";
|
|
26
26
|
import type { AdvisoryLockService } from "@checkstack/backend-api";
|
|
27
27
|
import type {
|
|
28
28
|
EntityChangeDeriver,
|
|
@@ -43,7 +43,13 @@ export const HEALTH_ENTITY_KIND = "health";
|
|
|
43
43
|
* same durable data `getSystemHealthStatus` reads — never materialized.
|
|
44
44
|
*/
|
|
45
45
|
export const HealthEntityStateSchema = z.object({
|
|
46
|
-
|
|
46
|
+
/**
|
|
47
|
+
* `unknown` when the system's checks have produced no runs to evaluate.
|
|
48
|
+
* Automations matching on `unhealthy` therefore do NOT fire for a system that
|
|
49
|
+
* is merely unmeasured - which is the point: an unmeasured system is not a
|
|
50
|
+
* detected outage, and inventing `healthy` for it would hide the gap instead.
|
|
51
|
+
*/
|
|
52
|
+
status: SystemHealthStatusSchema,
|
|
47
53
|
healthyChecks: z.number().int().nonnegative(),
|
|
48
54
|
totalChecks: z.number().int().nonnegative(),
|
|
49
55
|
});
|
package/src/health-state.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { and, desc, eq, gte, isNull } from "drizzle-orm";
|
|
2
|
-
import type {
|
|
2
|
+
import type { SystemHealthStatus } from "@checkstack/healthcheck-common";
|
|
3
3
|
import type { Logger, SafeDatabase } from "@checkstack/backend-api";
|
|
4
4
|
import type { InferClient } from "@checkstack/common";
|
|
5
5
|
import { MaintenanceApi } from "@checkstack/maintenance-common";
|
|
@@ -20,8 +20,11 @@ type MaintenanceClient = InferClient<typeof MaintenanceApi>;
|
|
|
20
20
|
* re-deriving the math each time.
|
|
21
21
|
*/
|
|
22
22
|
export interface HealthState {
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Aggregate status across all enabled checks. `unknown` when nothing has been
|
|
25
|
+
* measured yet - the system has no checks, or none of them has ever run.
|
|
26
|
+
*/
|
|
27
|
+
status: SystemHealthStatus;
|
|
25
28
|
/**
|
|
26
29
|
* When the system most recently entered `status`. Null when no
|
|
27
30
|
* transition has been recorded yet (fail-safe: never throws).
|
|
@@ -58,7 +61,8 @@ export interface HealthState {
|
|
|
58
61
|
|
|
59
62
|
/** Raw inputs to the pure builder, decoupled from the DB layer. */
|
|
60
63
|
export interface HealthStateInputs {
|
|
61
|
-
|
|
64
|
+
/** May be `unknown` - nothing measured yet. */
|
|
65
|
+
status: SystemHealthStatus;
|
|
62
66
|
inStatusSince: Date | null;
|
|
63
67
|
latencyMs?: number;
|
|
64
68
|
avgLatencyMs?: number;
|
|
@@ -330,7 +334,7 @@ export async function computeHealthState({
|
|
|
330
334
|
*/
|
|
331
335
|
environmentId?: string | null;
|
|
332
336
|
/** Returns the aggregate status for the system (per-check when scoped). */
|
|
333
|
-
resolveStatus: () => Promise<
|
|
337
|
+
resolveStatus: () => Promise<SystemHealthStatus>;
|
|
334
338
|
maintenanceClient?: MaintenanceClient;
|
|
335
339
|
logger?: Logger;
|
|
336
340
|
/** Trailing window (minutes) for the transition count. */
|
|
@@ -341,7 +345,12 @@ export async function computeHealthState({
|
|
|
341
345
|
|
|
342
346
|
const [inStatusSince, latest, windowed, inMaintenance, transitionsInWindow] =
|
|
343
347
|
await Promise.all([
|
|
344
|
-
|
|
348
|
+
// An UNMEASURED system has no transition rows to find - and `unknown` is
|
|
349
|
+
// not a member of the transition table's status enum, so querying for it
|
|
350
|
+
// would be a database error, not merely an empty result.
|
|
351
|
+
status === "unknown"
|
|
352
|
+
? Promise.resolve(null)
|
|
353
|
+
: findInStatusSince({ db, systemId, status, environmentId }),
|
|
345
354
|
findLatestRun({ db, systemId, configurationId, environmentId }),
|
|
346
355
|
computeWindowedMetrics({ db, systemId, configurationId, environmentId, now }),
|
|
347
356
|
resolveInMaintenance({ maintenanceClient, systemId, logger }),
|
package/src/index.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
setupHealthCheckWorker,
|
|
3
3
|
recomputeSystemRollupHealth,
|
|
4
|
+
persistRunAndReact,
|
|
4
5
|
} from "./queue-executor";
|
|
5
6
|
import { reconcileHealthCheckJobs } from "./schedule-reconciler";
|
|
7
|
+
import { SatelliteApi } from "@checkstack/satellite-common";
|
|
6
8
|
import { setupRetentionJob } from "./retention-job";
|
|
7
9
|
import * as schema from "./schema";
|
|
8
10
|
import {
|
|
@@ -522,6 +524,15 @@ export default createBackendPlugin({
|
|
|
522
524
|
cache,
|
|
523
525
|
secretResolver,
|
|
524
526
|
internalSecrets,
|
|
527
|
+
// Lets a satellite-ONLY check notice that none of its satellites are
|
|
528
|
+
// online and record a degraded run, instead of silently recording
|
|
529
|
+
// nothing and leaving its last status on screen forever. Resolved by
|
|
530
|
+
// RPC so this plugin keeps no direct dependency on satellite-backend.
|
|
531
|
+
getOnlineSatelliteIds: async () => {
|
|
532
|
+
const satelliteClient = rpcClient.forPlugin(SatelliteApi);
|
|
533
|
+
const result = await satelliteClient.getOnlineSatelliteIds();
|
|
534
|
+
return result.satelliteIds;
|
|
535
|
+
},
|
|
525
536
|
});
|
|
526
537
|
|
|
527
538
|
// Setup retention job for tiered storage (daily aggregation)
|
|
@@ -592,6 +603,28 @@ export default createBackendPlugin({
|
|
|
592
603
|
logger,
|
|
593
604
|
});
|
|
594
605
|
},
|
|
606
|
+
// Bind the service deps ONCE so an ingested satellite result drives
|
|
607
|
+
// the exact same post-run path (`persistRunAndReact`) as a local run:
|
|
608
|
+
// reactive entity write, cache reconcile, signals, automation hooks,
|
|
609
|
+
// transition, and notification. The router only supplies the per-run
|
|
610
|
+
// fields, so the two callers cannot drift.
|
|
611
|
+
reactToSatelliteRun: (run) =>
|
|
612
|
+
persistRunAndReact({
|
|
613
|
+
db: database,
|
|
614
|
+
service,
|
|
615
|
+
cache,
|
|
616
|
+
signalService,
|
|
617
|
+
notificationClient,
|
|
618
|
+
catalogClient,
|
|
619
|
+
maintenanceClient,
|
|
620
|
+
incidentClient,
|
|
621
|
+
getHealthEntity: () => healthEntity,
|
|
622
|
+
getEmitHook: () => storedEmitHook,
|
|
623
|
+
collectorRegistry,
|
|
624
|
+
advisoryLock,
|
|
625
|
+
logger,
|
|
626
|
+
...run,
|
|
627
|
+
}),
|
|
595
628
|
});
|
|
596
629
|
rpc.registerRouter(healthCheckRouter, healthCheckContract);
|
|
597
630
|
|
|
@@ -336,6 +336,276 @@ describe("Queue-Based Health Check Executor", () => {
|
|
|
336
336
|
});
|
|
337
337
|
});
|
|
338
338
|
|
|
339
|
+
describe("executeHealthCheckJob - satellite-only checks with no online satellite", () => {
|
|
340
|
+
/**
|
|
341
|
+
* Builds a worker whose one configuration is satellite-ONLY (includeLocal
|
|
342
|
+
* false, satellites assigned) and captures the queue handler.
|
|
343
|
+
*/
|
|
344
|
+
const setupSatelliteOnlyWorker = async (opts: {
|
|
345
|
+
getOnlineSatelliteIds?: () => Promise<string[]>;
|
|
346
|
+
/** Override the assignment shape to model a configuration change. */
|
|
347
|
+
configRow?: Partial<{
|
|
348
|
+
includeLocal: boolean;
|
|
349
|
+
satelliteIds: string[];
|
|
350
|
+
paused: boolean;
|
|
351
|
+
}>;
|
|
352
|
+
/** Records every subscriber notification the run would deliver. */
|
|
353
|
+
onNotify?: (input: unknown) => void;
|
|
354
|
+
}) => {
|
|
355
|
+
const mockDb = createMockDb();
|
|
356
|
+
const mockLogger = createMockLogger();
|
|
357
|
+
const mockQueueManager = createMockQueueManager();
|
|
358
|
+
const mockSignalService = createMockSignalService();
|
|
359
|
+
const mockCatalogClient = createMockCatalogClient();
|
|
360
|
+
|
|
361
|
+
// The shared mock has no `selectDistinct`; the rollup read uses it.
|
|
362
|
+
(mockDb as any).selectDistinct = mock(() => ({
|
|
363
|
+
from: mock(() => ({
|
|
364
|
+
where: mock(() => Promise.resolve([])),
|
|
365
|
+
})),
|
|
366
|
+
}));
|
|
367
|
+
|
|
368
|
+
(mockDb.select as any) = mock(() => ({
|
|
369
|
+
from: mock(() => ({
|
|
370
|
+
innerJoin: mock(() => ({
|
|
371
|
+
where: mock(() =>
|
|
372
|
+
Promise.resolve([
|
|
373
|
+
{
|
|
374
|
+
configId: "config-1",
|
|
375
|
+
configName: "Satellite check",
|
|
376
|
+
strategyId: "test-strategy",
|
|
377
|
+
config: {},
|
|
378
|
+
collectors: [],
|
|
379
|
+
interval: 30,
|
|
380
|
+
enabled: true,
|
|
381
|
+
// The shape this branch exists for: executed by satellites,
|
|
382
|
+
// never by the core. Overridable so a test can model an
|
|
383
|
+
// assignment change.
|
|
384
|
+
paused: false,
|
|
385
|
+
includeLocal: false,
|
|
386
|
+
satelliteIds: ["sat-1", "sat-2"],
|
|
387
|
+
environmentIds: null,
|
|
388
|
+
...opts.configRow,
|
|
389
|
+
},
|
|
390
|
+
]),
|
|
391
|
+
),
|
|
392
|
+
})),
|
|
393
|
+
where: mock(() => Promise.resolve([])),
|
|
394
|
+
})),
|
|
395
|
+
}));
|
|
396
|
+
|
|
397
|
+
const queue =
|
|
398
|
+
mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
|
|
399
|
+
let capturedHandler:
|
|
400
|
+
| ((job: { data: HealthCheckJobPayload }) => Promise<void>)
|
|
401
|
+
| undefined;
|
|
402
|
+
(queue.consume as any) = mock(
|
|
403
|
+
async (
|
|
404
|
+
handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
|
|
405
|
+
) => {
|
|
406
|
+
capturedHandler = handler;
|
|
407
|
+
},
|
|
408
|
+
);
|
|
409
|
+
|
|
410
|
+
await setupHealthCheckWorker({
|
|
411
|
+
db: mockDb as unknown as Parameters<
|
|
412
|
+
typeof setupHealthCheckWorker
|
|
413
|
+
>[0]["db"],
|
|
414
|
+
advisoryLock: mockAdvisoryLock,
|
|
415
|
+
registry: createMockRegistry(),
|
|
416
|
+
collectorRegistry:
|
|
417
|
+
createMockCollectorRegistry() as unknown as Parameters<
|
|
418
|
+
typeof setupHealthCheckWorker
|
|
419
|
+
>[0]["collectorRegistry"],
|
|
420
|
+
logger: mockLogger,
|
|
421
|
+
queueManager: mockQueueManager,
|
|
422
|
+
signalService: mockSignalService,
|
|
423
|
+
catalogClient: mockCatalogClient as unknown as Parameters<
|
|
424
|
+
typeof setupHealthCheckWorker
|
|
425
|
+
>[0]["catalogClient"],
|
|
426
|
+
notificationClient: {
|
|
427
|
+
notifyForSubscription: (input: unknown) => {
|
|
428
|
+
opts.onNotify?.(input);
|
|
429
|
+
return Promise.resolve({ notifiedCount: 0 });
|
|
430
|
+
},
|
|
431
|
+
} as unknown as Parameters<
|
|
432
|
+
typeof setupHealthCheckWorker
|
|
433
|
+
>[0]["notificationClient"],
|
|
434
|
+
maintenanceClient: createMockMaintenanceClient() as unknown as Parameters<
|
|
435
|
+
typeof setupHealthCheckWorker
|
|
436
|
+
>[0]["maintenanceClient"],
|
|
437
|
+
incidentClient: createMockIncidentClient() as unknown as Parameters<
|
|
438
|
+
typeof setupHealthCheckWorker
|
|
439
|
+
>[0]["incidentClient"],
|
|
440
|
+
getEmitHook: () => undefined,
|
|
441
|
+
cache: passthroughCache,
|
|
442
|
+
slowCheckRuntime: null,
|
|
443
|
+
...(opts.getOnlineSatelliteIds
|
|
444
|
+
? { getOnlineSatelliteIds: opts.getOnlineSatelliteIds }
|
|
445
|
+
: {}),
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
return { capturedHandler, mockLogger, mockSignalService };
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
const run = async (
|
|
452
|
+
handler:
|
|
453
|
+
| ((job: { data: HealthCheckJobPayload }) => Promise<void>)
|
|
454
|
+
| undefined,
|
|
455
|
+
) => {
|
|
456
|
+
await handler?.({
|
|
457
|
+
data: {
|
|
458
|
+
configId: "config-1",
|
|
459
|
+
systemId: "system-1",
|
|
460
|
+
environmentId: null,
|
|
461
|
+
},
|
|
462
|
+
});
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
it("stays silent while at least one assigned satellite is online", async () => {
|
|
466
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
467
|
+
getOnlineSatelliteIds: async () => ["sat-2"],
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
await run(capturedHandler);
|
|
471
|
+
|
|
472
|
+
expect(mockLogger.debug).toHaveBeenCalledWith(
|
|
473
|
+
expect.stringContaining("satellite-only, skipping local execution"),
|
|
474
|
+
);
|
|
475
|
+
expect(mockLogger.warn).not.toHaveBeenCalledWith(
|
|
476
|
+
expect.stringContaining("no online satellite"),
|
|
477
|
+
);
|
|
478
|
+
});
|
|
479
|
+
|
|
480
|
+
it("warns and records a run when NO assigned satellite is online", async () => {
|
|
481
|
+
// The regression this guards: the core used to return silently here, so a
|
|
482
|
+
// check whose satellites were all down kept displaying its last known
|
|
483
|
+
// status forever - a dead probe reading exactly like a passing one.
|
|
484
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
485
|
+
getOnlineSatelliteIds: async () => [],
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
// `persistRunAndReact` needs the full insert/aggregate/entity chain, which
|
|
489
|
+
// this file's mock database does not provide - so the call throws HERE and
|
|
490
|
+
// not in production, where it is the same function every successful run
|
|
491
|
+
// goes through. That is a mock limit, not the behaviour under test: what
|
|
492
|
+
// this pins is that the executor took the RECORD path instead of the
|
|
493
|
+
// silent one. What gets recorded is pinned separately and purely by
|
|
494
|
+
// `buildUnobservableRun` in `satellite-liveness.test.ts`.
|
|
495
|
+
await run(capturedHandler).catch(() => {});
|
|
496
|
+
|
|
497
|
+
expect(mockLogger.warn).toHaveBeenCalledWith(
|
|
498
|
+
expect.stringContaining("no online satellite"),
|
|
499
|
+
);
|
|
500
|
+
// It must NOT take the silent path.
|
|
501
|
+
expect(mockLogger.debug).not.toHaveBeenCalledWith(
|
|
502
|
+
expect.stringContaining("satellite-only, skipping local execution"),
|
|
503
|
+
);
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
it("stays silent when satellite liveness cannot be resolved", async () => {
|
|
507
|
+
// A transient failure to reach the satellite service must never mark every
|
|
508
|
+
// satellite-only check in the fleet degraded at once.
|
|
509
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
510
|
+
getOnlineSatelliteIds: async () => {
|
|
511
|
+
throw new Error("satellite service unreachable");
|
|
512
|
+
},
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
await run(capturedHandler);
|
|
516
|
+
|
|
517
|
+
expect(mockLogger.debug).toHaveBeenCalledWith(
|
|
518
|
+
expect.stringContaining("satellite-only, skipping local execution"),
|
|
519
|
+
);
|
|
520
|
+
expect(mockLogger.warn).not.toHaveBeenCalledWith(
|
|
521
|
+
expect.stringContaining("no online satellite"),
|
|
522
|
+
);
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it("stays silent when no liveness resolver is wired at all", async () => {
|
|
526
|
+
// Pre-existing behaviour for a deployment without the resolver: the core
|
|
527
|
+
// says nothing and lets the satellites report.
|
|
528
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({});
|
|
529
|
+
|
|
530
|
+
await run(capturedHandler);
|
|
531
|
+
|
|
532
|
+
expect(mockLogger.debug).toHaveBeenCalledWith(
|
|
533
|
+
expect.stringContaining("satellite-only, skipping local execution"),
|
|
534
|
+
);
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* An operator CHANGING an assignment must never look like a failure. These
|
|
539
|
+
* are the cases that have historically been got wrong: the platform reacts
|
|
540
|
+
* to a deliberate configuration change as though the check had broken.
|
|
541
|
+
*/
|
|
542
|
+
it("records nothing when the satellites are removed from the assignment", async () => {
|
|
543
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
544
|
+
// No satellite is online at all - but the assignment no longer names
|
|
545
|
+
// any, so this check is simply not satellite-only any more.
|
|
546
|
+
getOnlineSatelliteIds: async () => [],
|
|
547
|
+
configRow: { satelliteIds: [] },
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
await run(capturedHandler).catch(() => {});
|
|
551
|
+
|
|
552
|
+
expect(mockLogger.warn).not.toHaveBeenCalledWith(
|
|
553
|
+
expect.stringContaining("no online satellite"),
|
|
554
|
+
);
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
it("records nothing when local execution is turned back on", async () => {
|
|
558
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
559
|
+
getOnlineSatelliteIds: async () => [],
|
|
560
|
+
configRow: { includeLocal: true },
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
await run(capturedHandler).catch(() => {});
|
|
564
|
+
|
|
565
|
+
expect(mockLogger.warn).not.toHaveBeenCalledWith(
|
|
566
|
+
expect.stringContaining("no online satellite"),
|
|
567
|
+
);
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it("a PAUSED satellite-only check records nothing, even with every satellite offline", async () => {
|
|
571
|
+
// Paused is checked BEFORE the satellite branch, so a paused check is
|
|
572
|
+
// quiet on purpose and must not manufacture a degraded run.
|
|
573
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
574
|
+
getOnlineSatelliteIds: async () => [],
|
|
575
|
+
configRow: { paused: true },
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
await run(capturedHandler).catch(() => {});
|
|
579
|
+
|
|
580
|
+
expect(mockLogger.debug).toHaveBeenCalledWith(
|
|
581
|
+
expect.stringContaining("is paused, skipping execution"),
|
|
582
|
+
);
|
|
583
|
+
expect(mockLogger.warn).not.toHaveBeenCalledWith(
|
|
584
|
+
expect.stringContaining("no online satellite"),
|
|
585
|
+
);
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
it("does NOT notify subscribers for the unobservable run", async () => {
|
|
589
|
+
// One offline satellite degrades EVERY check assigned to it in the same
|
|
590
|
+
// tick, and healthy -> degraded is an escalation, so notifying per check
|
|
591
|
+
// turns a single root cause into one alert per check. The satellite's own
|
|
592
|
+
// connectivity subscription reports the cause once. The RUN is still
|
|
593
|
+
// recorded - only the per-check alert is withheld.
|
|
594
|
+
const notified: unknown[] = [];
|
|
595
|
+
const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
|
|
596
|
+
getOnlineSatelliteIds: async () => [],
|
|
597
|
+
onNotify: (input) => notified.push(input),
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
await run(capturedHandler).catch(() => {});
|
|
601
|
+
|
|
602
|
+
expect(mockLogger.warn).toHaveBeenCalledWith(
|
|
603
|
+
expect.stringContaining("no online satellite"),
|
|
604
|
+
);
|
|
605
|
+
expect(notified).toHaveLength(0);
|
|
606
|
+
});
|
|
607
|
+
});
|
|
608
|
+
|
|
339
609
|
describe("executeHealthCheckJob - collector run-context", () => {
|
|
340
610
|
it("passes curated run-context to the collector (name falls back to id when configName is null)", async () => {
|
|
341
611
|
const mockDb = createMockDb();
|