@checkstack/healthcheck-backend 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -0
- package/package.json +29 -29
- package/src/aggregation-utils.test.ts +132 -0
- package/src/aggregation-utils.ts +70 -6
- package/src/ai/healthcheck-propose.ts +9 -0
- package/src/ai/healthcheck-update.ts +5 -1
- package/src/collector-assertions.test.ts +97 -1
- package/src/collector-assertions.ts +141 -34
- package/src/config-secrets-backfill.test.ts +142 -0
- package/src/config-secrets-backfill.ts +129 -0
- package/src/config-secrets.test.ts +771 -0
- package/src/config-secrets.ts +361 -0
- package/src/healthcheck-gitops-kinds.ts +42 -31
- package/src/index.ts +47 -1
- package/src/queue-executor.test.ts +182 -0
- package/src/queue-executor.ts +64 -6
- package/src/realtime-aggregation.test.ts +136 -0
- package/src/realtime-aggregation.ts +39 -2
- package/src/retention-job.ts +64 -1
- package/src/retention-rollup.test.ts +69 -0
- package/src/router-config-secrets.test.ts +165 -0
- package/src/router-create-and-assign.test.ts +5 -1
- package/src/router.ts +39 -7
- package/src/service-config-secrets.test.ts +680 -0
- package/src/service-ingest-assertions.test.ts +213 -0
- package/src/service.ts +529 -56
|
@@ -1459,3 +1459,185 @@ describe("recomputeSystemRollupHealth", () => {
|
|
|
1459
1459
|
expect(logger.error).toHaveBeenCalledTimes(1);
|
|
1460
1460
|
});
|
|
1461
1461
|
});
|
|
1462
|
+
|
|
1463
|
+
describe("executeHealthCheckJob - structured assertion outcomes", () => {
|
|
1464
|
+
it("stores _assertions per collector entry and downgrades on failure", async () => {
|
|
1465
|
+
const mockDb = createMockDb();
|
|
1466
|
+
const mockRegistry = createMockRegistry();
|
|
1467
|
+
const mockLogger = createMockLogger();
|
|
1468
|
+
const mockQueueManager = createMockQueueManager();
|
|
1469
|
+
const mockCatalogClient = createMockCatalogClient();
|
|
1470
|
+
const mockMaintenanceClient = createMockMaintenanceClient();
|
|
1471
|
+
const mockIncidentClient = createMockIncidentClient();
|
|
1472
|
+
const mockSignalService = createMockSignalService();
|
|
1473
|
+
|
|
1474
|
+
(mockCatalogClient.getSystem as any) = mock(async () => ({
|
|
1475
|
+
id: "system-1",
|
|
1476
|
+
name: "web-01",
|
|
1477
|
+
}));
|
|
1478
|
+
|
|
1479
|
+
// One collector entry with two assertions: isTrue passes, equals fails.
|
|
1480
|
+
let selectCallCount = 0;
|
|
1481
|
+
(mockDb.select as any) = mock(() => {
|
|
1482
|
+
selectCallCount++;
|
|
1483
|
+
if (selectCallCount === 2) {
|
|
1484
|
+
return {
|
|
1485
|
+
from: mock(() => ({
|
|
1486
|
+
innerJoin: mock(() => ({
|
|
1487
|
+
where: mock(() =>
|
|
1488
|
+
Promise.resolve([
|
|
1489
|
+
{
|
|
1490
|
+
configId: "config-1",
|
|
1491
|
+
configName: "checkout",
|
|
1492
|
+
strategyId: "test-strategy",
|
|
1493
|
+
config: { timeout: 5000 },
|
|
1494
|
+
collectors: [
|
|
1495
|
+
{
|
|
1496
|
+
id: "col-1",
|
|
1497
|
+
collectorId: "test-collector",
|
|
1498
|
+
config: {},
|
|
1499
|
+
assertions: [
|
|
1500
|
+
{ field: "ok", operator: "isTrue" },
|
|
1501
|
+
{ field: "code", operator: "equals", value: 200 },
|
|
1502
|
+
],
|
|
1503
|
+
},
|
|
1504
|
+
],
|
|
1505
|
+
interval: 45,
|
|
1506
|
+
enabled: true,
|
|
1507
|
+
paused: false,
|
|
1508
|
+
includeLocal: true,
|
|
1509
|
+
satelliteIds: [],
|
|
1510
|
+
},
|
|
1511
|
+
]),
|
|
1512
|
+
),
|
|
1513
|
+
})),
|
|
1514
|
+
})),
|
|
1515
|
+
};
|
|
1516
|
+
}
|
|
1517
|
+
return {
|
|
1518
|
+
from: mock(() => ({
|
|
1519
|
+
innerJoin: mock(() => ({
|
|
1520
|
+
where: mock(() => Promise.resolve([])),
|
|
1521
|
+
})),
|
|
1522
|
+
})),
|
|
1523
|
+
};
|
|
1524
|
+
});
|
|
1525
|
+
|
|
1526
|
+
// Capture every insert's values so we can find the stored run.
|
|
1527
|
+
const insertedValues: Record<string, unknown>[] = [];
|
|
1528
|
+
(mockDb.insert as any) = mock(() => ({
|
|
1529
|
+
values: mock((vals: Record<string, unknown>) => {
|
|
1530
|
+
insertedValues.push(vals);
|
|
1531
|
+
return Object.assign(Promise.resolve(), {
|
|
1532
|
+
onConflictDoUpdate: mock(() => Promise.resolve()),
|
|
1533
|
+
onConflictDoNothing: mock(() => Promise.resolve()),
|
|
1534
|
+
returning: mock(() => Promise.resolve([])),
|
|
1535
|
+
});
|
|
1536
|
+
}),
|
|
1537
|
+
}));
|
|
1538
|
+
|
|
1539
|
+
const mockCollectorRegistry = {
|
|
1540
|
+
register: mock(() => {}),
|
|
1541
|
+
getCollector: mock(() => ({
|
|
1542
|
+
collector: {
|
|
1543
|
+
id: "test-collector",
|
|
1544
|
+
execute: mock(async () => ({ result: { ok: true, code: 404 } })),
|
|
1545
|
+
config: new Versioned({ version: 1, schema: z.object({}) }),
|
|
1546
|
+
result: new Versioned({
|
|
1547
|
+
version: 1,
|
|
1548
|
+
schema: z.object({ ok: z.boolean(), code: z.number() }),
|
|
1549
|
+
}),
|
|
1550
|
+
mergeResult: mock(() => ({})),
|
|
1551
|
+
},
|
|
1552
|
+
})),
|
|
1553
|
+
getCollectors: mock(() => []),
|
|
1554
|
+
};
|
|
1555
|
+
|
|
1556
|
+
const queue =
|
|
1557
|
+
mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
|
|
1558
|
+
let capturedHandler:
|
|
1559
|
+
| ((job: { data: HealthCheckJobPayload }) => Promise<void>)
|
|
1560
|
+
| undefined;
|
|
1561
|
+
(queue.consume as any) = mock(
|
|
1562
|
+
async (
|
|
1563
|
+
handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
|
|
1564
|
+
) => {
|
|
1565
|
+
capturedHandler = handler;
|
|
1566
|
+
},
|
|
1567
|
+
);
|
|
1568
|
+
|
|
1569
|
+
await setupHealthCheckWorker({
|
|
1570
|
+
db: mockDb as unknown as Parameters<
|
|
1571
|
+
typeof setupHealthCheckWorker
|
|
1572
|
+
>[0]["db"],
|
|
1573
|
+
advisoryLock: mockAdvisoryLock,
|
|
1574
|
+
registry: mockRegistry,
|
|
1575
|
+
collectorRegistry: mockCollectorRegistry as unknown as Parameters<
|
|
1576
|
+
typeof setupHealthCheckWorker
|
|
1577
|
+
>[0]["collectorRegistry"],
|
|
1578
|
+
logger: mockLogger,
|
|
1579
|
+
queueManager: mockQueueManager,
|
|
1580
|
+
signalService: mockSignalService,
|
|
1581
|
+
catalogClient: mockCatalogClient as unknown as Parameters<
|
|
1582
|
+
typeof setupHealthCheckWorker
|
|
1583
|
+
>[0]["catalogClient"],
|
|
1584
|
+
notificationClient: {
|
|
1585
|
+
notifyForSubscription: () => Promise.resolve({ notifiedCount: 0 }),
|
|
1586
|
+
} as unknown as Parameters<
|
|
1587
|
+
typeof setupHealthCheckWorker
|
|
1588
|
+
>[0]["notificationClient"],
|
|
1589
|
+
maintenanceClient: mockMaintenanceClient as unknown as Parameters<
|
|
1590
|
+
typeof setupHealthCheckWorker
|
|
1591
|
+
>[0]["maintenanceClient"],
|
|
1592
|
+
incidentClient: mockIncidentClient as unknown as Parameters<
|
|
1593
|
+
typeof setupHealthCheckWorker
|
|
1594
|
+
>[0]["incidentClient"],
|
|
1595
|
+
getEmitHook: () => undefined,
|
|
1596
|
+
cache: passthroughCache,
|
|
1597
|
+
});
|
|
1598
|
+
|
|
1599
|
+
if (capturedHandler) {
|
|
1600
|
+
// Downstream aggregation touches DB surfaces the lightweight mock
|
|
1601
|
+
// doesn't model; the run insert we assert on happens before that.
|
|
1602
|
+
await capturedHandler({
|
|
1603
|
+
data: { configId: "config-1", systemId: "system-1" },
|
|
1604
|
+
}).catch(() => {});
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
const runInsert = insertedValues.find(
|
|
1608
|
+
(vals) => "status" in vals && "result" in vals,
|
|
1609
|
+
);
|
|
1610
|
+
expect(runInsert).toBeDefined();
|
|
1611
|
+
// The failed `code equals 200` assertion downgrades the run.
|
|
1612
|
+
expect(runInsert?.status).toBe("unhealthy");
|
|
1613
|
+
|
|
1614
|
+
const runResult = runInsert?.result as {
|
|
1615
|
+
metadata: {
|
|
1616
|
+
collectors: Record<
|
|
1617
|
+
string,
|
|
1618
|
+
{
|
|
1619
|
+
_assertionFailed?: string;
|
|
1620
|
+
_assertions?: {
|
|
1621
|
+
field: string;
|
|
1622
|
+
passed: boolean;
|
|
1623
|
+
actual?: string;
|
|
1624
|
+
}[];
|
|
1625
|
+
}
|
|
1626
|
+
>;
|
|
1627
|
+
};
|
|
1628
|
+
};
|
|
1629
|
+
const entry = runResult.metadata.collectors["col-1"];
|
|
1630
|
+
expect(entry._assertionFailed).toBe("code equals 200");
|
|
1631
|
+
expect(entry._assertions?.length).toBe(2);
|
|
1632
|
+
expect(entry._assertions?.[0]).toMatchObject({
|
|
1633
|
+
field: "ok",
|
|
1634
|
+
passed: true,
|
|
1635
|
+
actual: "true",
|
|
1636
|
+
});
|
|
1637
|
+
expect(entry._assertions?.[1]).toMatchObject({
|
|
1638
|
+
field: "code",
|
|
1639
|
+
passed: false,
|
|
1640
|
+
actual: "404",
|
|
1641
|
+
});
|
|
1642
|
+
});
|
|
1643
|
+
});
|
package/src/queue-executor.ts
CHANGED
|
@@ -46,7 +46,11 @@ import { NotificationApi } from "@checkstack/notification-common";
|
|
|
46
46
|
import { healthcheckSystemSubscription } from "@checkstack/healthcheck-common";
|
|
47
47
|
import { resolveRoute, type InferClient, extractErrorMessage} from "@checkstack/common";
|
|
48
48
|
import { secretEnvMappingSchema } from "@checkstack/secrets-common";
|
|
49
|
-
import type {
|
|
49
|
+
import type {
|
|
50
|
+
SecretResolverService,
|
|
51
|
+
InternalSecretsService,
|
|
52
|
+
} from "@checkstack/secrets-backend";
|
|
53
|
+
import { inflateConfigSecrets } from "./config-secrets";
|
|
50
54
|
import { HealthCheckService } from "./service";
|
|
51
55
|
import { healthCheckHooks } from "./hooks";
|
|
52
56
|
import { incrementHourlyAggregate } from "./realtime-aggregation";
|
|
@@ -56,7 +60,8 @@ import {
|
|
|
56
60
|
shouldNotifyTransition,
|
|
57
61
|
} from "./notification-policy";
|
|
58
62
|
import { recordStateTransition } from "./state-transitions";
|
|
59
|
-
import {
|
|
63
|
+
import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
|
|
64
|
+
import type { AssertionOutcome } from "@checkstack/healthcheck-common";
|
|
60
65
|
import {
|
|
61
66
|
writeHealthEntity,
|
|
62
67
|
createHealthEntitySerializer,
|
|
@@ -538,6 +543,14 @@ async function executeHealthCheckJob(props: {
|
|
|
538
543
|
* / test isolation.
|
|
539
544
|
*/
|
|
540
545
|
secretResolver?: SecretResolverService;
|
|
546
|
+
/**
|
|
547
|
+
* Internal secret store. When set (together with `secretResolver`), stored
|
|
548
|
+
* strategy/collector config `x-secret` fields - internal markers and
|
|
549
|
+
* `${{ secrets.* }}` references - are INFLATED to their real values just
|
|
550
|
+
* before use, in memory only. Optional for version-skew / test isolation;
|
|
551
|
+
* without it, marker-bearing configs fail their runs clearly.
|
|
552
|
+
*/
|
|
553
|
+
internalSecrets?: InternalSecretsService;
|
|
541
554
|
}): Promise<void> {
|
|
542
555
|
const {
|
|
543
556
|
payload,
|
|
@@ -555,6 +568,7 @@ async function executeHealthCheckJob(props: {
|
|
|
555
568
|
getHealthEntity,
|
|
556
569
|
cache,
|
|
557
570
|
secretResolver,
|
|
571
|
+
internalSecrets,
|
|
558
572
|
} = props;
|
|
559
573
|
const { configId, systemId } = payload;
|
|
560
574
|
|
|
@@ -656,6 +670,22 @@ async function executeHealthCheckJob(props: {
|
|
|
656
670
|
return;
|
|
657
671
|
}
|
|
658
672
|
|
|
673
|
+
// Inflate stored secret markers / `${{ secrets.* }}` references to their
|
|
674
|
+
// real values ONCE, memory-only, BEFORE migrate+validate - so validation
|
|
675
|
+
// sees real values. Old-shape rows (whose current-schema secret keys do
|
|
676
|
+
// not exist yet) and legacy bare literals pass through untouched.
|
|
677
|
+
let rawStrategyConfig = configRow.config;
|
|
678
|
+
if (internalSecrets && secretResolver) {
|
|
679
|
+
const inflated = await inflateConfigSecrets({
|
|
680
|
+
configurationId: configId,
|
|
681
|
+
scope: { kind: "strategy" },
|
|
682
|
+
schema: strategy.config.schema,
|
|
683
|
+
config: configRow.config,
|
|
684
|
+
deps: { internalSecrets, secretResolver },
|
|
685
|
+
});
|
|
686
|
+
rawStrategyConfig = inflated.config;
|
|
687
|
+
}
|
|
688
|
+
|
|
659
689
|
// Migrate the stored (UNVERSIONED) strategy config ONCE, before the
|
|
660
690
|
// per-environment render loop, so every env renders from the same
|
|
661
691
|
// migrated shape. Stored configs predate explicit versioning and may be
|
|
@@ -663,7 +693,7 @@ async function executeHealthCheckJob(props: {
|
|
|
663
693
|
// -on-read runs the declared migration chain, then validates. The
|
|
664
694
|
// migrations are idempotent, so an already-current config is a no-op.
|
|
665
695
|
const strategyConfig: BaseStrategyConfig =
|
|
666
|
-
await strategy.config.parseAssumingV1(
|
|
696
|
+
await strategy.config.parseAssumingV1(rawStrategyConfig);
|
|
667
697
|
const executionTimeout = strategyConfig.timeout ?? 60_000;
|
|
668
698
|
|
|
669
699
|
// ── Per-environment fan-out (§7) ────────────────────────────────────────
|
|
@@ -872,9 +902,26 @@ async function executeHealthCheckJob(props: {
|
|
|
872
902
|
// reads the raw `secretEnv` mapping (a constant string field
|
|
873
903
|
// unaffected by the strategy/collector reshapes), keeping the
|
|
874
904
|
// migrate -> secret resolve -> render -> execute order intact.
|
|
905
|
+
// Inflate this entry's secret markers / references (memory
|
|
906
|
+
// only) before its migrate+validate parse, mirroring the
|
|
907
|
+
// strategy-config inflation above.
|
|
908
|
+
let rawCollectorConfig = collectorEntry.config;
|
|
909
|
+
if (internalSecrets && secretResolver) {
|
|
910
|
+
const inflated = await inflateConfigSecrets({
|
|
911
|
+
configurationId: configId,
|
|
912
|
+
scope: {
|
|
913
|
+
kind: "collector",
|
|
914
|
+
entryId: collectorEntry.id,
|
|
915
|
+
},
|
|
916
|
+
schema: registered.collector.config.schema,
|
|
917
|
+
config: collectorEntry.config,
|
|
918
|
+
deps: { internalSecrets, secretResolver },
|
|
919
|
+
});
|
|
920
|
+
rawCollectorConfig = inflated.config;
|
|
921
|
+
}
|
|
875
922
|
const migratedCollectorConfig =
|
|
876
923
|
await registered.collector.config.parseAssumingV1(
|
|
877
|
-
|
|
924
|
+
rawCollectorConfig,
|
|
878
925
|
);
|
|
879
926
|
|
|
880
927
|
// (2) Environment/templating pass for the collector config -
|
|
@@ -901,13 +948,18 @@ async function executeHealthCheckJob(props: {
|
|
|
901
948
|
collectorError = collectorResult.error;
|
|
902
949
|
}
|
|
903
950
|
|
|
904
|
-
// Evaluate per-collector assertions (plain fields + JSONPath)
|
|
951
|
+
// Evaluate per-collector assertions (plain fields + JSONPath).
|
|
952
|
+
// ALL outcomes are stored (pass AND fail) so assertions are
|
|
953
|
+
// analyzable over time, not only visible on failure.
|
|
905
954
|
let assertionFailed: string | undefined;
|
|
955
|
+
let assertionOutcomes: AssertionOutcome[] = [];
|
|
906
956
|
if (collectorResult.result) {
|
|
907
|
-
|
|
957
|
+
const evaluation = evaluateCollectorAssertionOutcomes({
|
|
908
958
|
assertions: collectorEntry.assertions,
|
|
909
959
|
result: collectorResult.result as Record<string, unknown>,
|
|
910
960
|
});
|
|
961
|
+
assertionFailed = evaluation.firstFailureMessage;
|
|
962
|
+
assertionOutcomes = evaluation.outcomes;
|
|
911
963
|
if (assertionFailed) {
|
|
912
964
|
logger.debug(
|
|
913
965
|
`Collector ${storageKey} assertion failed: ${assertionFailed}`,
|
|
@@ -931,6 +983,9 @@ async function executeHealthCheckJob(props: {
|
|
|
931
983
|
_collectorId: collectorEntry.collectorId,
|
|
932
984
|
_assertionFailed: assertionFailed,
|
|
933
985
|
_collectorError: collectorError,
|
|
986
|
+
...(assertionOutcomes.length > 0
|
|
987
|
+
? { _assertions: assertionOutcomes }
|
|
988
|
+
: {}),
|
|
934
989
|
...strippedResult,
|
|
935
990
|
},
|
|
936
991
|
};
|
|
@@ -1518,6 +1573,7 @@ export async function setupHealthCheckWorker(props: {
|
|
|
1518
1573
|
getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
|
|
1519
1574
|
cache: HealthCheckCache;
|
|
1520
1575
|
secretResolver?: SecretResolverService;
|
|
1576
|
+
internalSecrets?: InternalSecretsService;
|
|
1521
1577
|
}): Promise<void> {
|
|
1522
1578
|
const {
|
|
1523
1579
|
db,
|
|
@@ -1535,6 +1591,7 @@ export async function setupHealthCheckWorker(props: {
|
|
|
1535
1591
|
getHealthEntity,
|
|
1536
1592
|
cache,
|
|
1537
1593
|
secretResolver,
|
|
1594
|
+
internalSecrets,
|
|
1538
1595
|
} = props;
|
|
1539
1596
|
|
|
1540
1597
|
const queue =
|
|
@@ -1559,6 +1616,7 @@ export async function setupHealthCheckWorker(props: {
|
|
|
1559
1616
|
getHealthEntity,
|
|
1560
1617
|
cache,
|
|
1561
1618
|
secretResolver,
|
|
1619
|
+
internalSecrets,
|
|
1562
1620
|
});
|
|
1563
1621
|
},
|
|
1564
1622
|
{
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
deserializeTDigest,
|
|
6
6
|
incrementHourlyAggregate,
|
|
7
7
|
} from "./realtime-aggregation";
|
|
8
|
+
import { computeAssertionKey } from "@checkstack/healthcheck-common";
|
|
8
9
|
import { TDigest } from "tdigest";
|
|
9
10
|
|
|
10
11
|
describe("getHourBucketStart", () => {
|
|
@@ -494,3 +495,138 @@ describe("incrementHourlyAggregate", () => {
|
|
|
494
495
|
expect(inserted.maxLatencyMs).toBe(150);
|
|
495
496
|
});
|
|
496
497
|
});
|
|
498
|
+
|
|
499
|
+
describe("incrementHourlyAggregate - assertion stats folding", () => {
|
|
500
|
+
const KEY = computeAssertionKey({
|
|
501
|
+
assertion: { field: "statusCode", operator: "equals", value: 200 },
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
const outcome = (passed: boolean) => ({
|
|
505
|
+
key: KEY,
|
|
506
|
+
field: "statusCode",
|
|
507
|
+
operator: "equals",
|
|
508
|
+
value: "200",
|
|
509
|
+
passed,
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
const collectorRegistry = {
|
|
513
|
+
register: mock(() => {}),
|
|
514
|
+
getCollector: mock(() => ({
|
|
515
|
+
collector: {
|
|
516
|
+
id: "test-collector",
|
|
517
|
+
mergeResult: mock(() => ({ count: 1 })),
|
|
518
|
+
},
|
|
519
|
+
})),
|
|
520
|
+
getCollectors: mock(() => []),
|
|
521
|
+
} as unknown as Parameters<
|
|
522
|
+
typeof incrementHourlyAggregate
|
|
523
|
+
>[0]["collectorRegistry"];
|
|
524
|
+
|
|
525
|
+
function runResult(outcomes: unknown[]) {
|
|
526
|
+
return {
|
|
527
|
+
metadata: {
|
|
528
|
+
collectors: {
|
|
529
|
+
"uuid-1": {
|
|
530
|
+
_collectorId: "test-collector",
|
|
531
|
+
_assertions: outcomes,
|
|
532
|
+
statusCode: 200,
|
|
533
|
+
},
|
|
534
|
+
},
|
|
535
|
+
},
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
it("folds a run's outcomes into the bucket's per-assertion counts", async () => {
|
|
540
|
+
let inserted: Record<string, unknown> | undefined;
|
|
541
|
+
const db = {
|
|
542
|
+
select: mock(() => ({
|
|
543
|
+
from: mock(() => ({
|
|
544
|
+
where: mock(() => ({
|
|
545
|
+
limit: mock(() => Promise.resolve([])),
|
|
546
|
+
})),
|
|
547
|
+
})),
|
|
548
|
+
})),
|
|
549
|
+
insert: mock(() => ({
|
|
550
|
+
values: mock((values: Record<string, unknown>) => {
|
|
551
|
+
inserted = values;
|
|
552
|
+
return { onConflictDoUpdate: mock(() => Promise.resolve()) };
|
|
553
|
+
}),
|
|
554
|
+
})),
|
|
555
|
+
};
|
|
556
|
+
|
|
557
|
+
await incrementHourlyAggregate({
|
|
558
|
+
db: db as never,
|
|
559
|
+
systemId: "sys-1",
|
|
560
|
+
configurationId: "config-1",
|
|
561
|
+
status: "unhealthy",
|
|
562
|
+
latencyMs: 100,
|
|
563
|
+
runTimestamp: new Date("2024-01-15T10:35:00Z"),
|
|
564
|
+
result: runResult([outcome(true), outcome(false)]) as never,
|
|
565
|
+
collectorRegistry,
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
|
|
569
|
+
expect(aggregated.assertions).toEqual({
|
|
570
|
+
"uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
|
|
571
|
+
});
|
|
572
|
+
// The internal _assertions never leak into collector merge output.
|
|
573
|
+
const collectors = aggregated.collectors as Record<
|
|
574
|
+
string,
|
|
575
|
+
Record<string, unknown>
|
|
576
|
+
>;
|
|
577
|
+
expect(collectors["uuid-1"]._assertions).toBeUndefined();
|
|
578
|
+
});
|
|
579
|
+
|
|
580
|
+
it("increments existing counts and keeps distinct keys separate (mid-bucket config edit)", async () => {
|
|
581
|
+
const OTHER_KEY = computeAssertionKey({
|
|
582
|
+
assertion: { field: "statusCode", operator: "equals", value: 201 },
|
|
583
|
+
});
|
|
584
|
+
const existing = {
|
|
585
|
+
tdigestState: null,
|
|
586
|
+
minLatencyMs: null,
|
|
587
|
+
maxLatencyMs: null,
|
|
588
|
+
aggregatedResult: {
|
|
589
|
+
collectors: {},
|
|
590
|
+
assertions: { "uuid-1": { [KEY]: { passCount: 4, failCount: 0 } } },
|
|
591
|
+
},
|
|
592
|
+
};
|
|
593
|
+
let inserted: Record<string, unknown> | undefined;
|
|
594
|
+
const db = {
|
|
595
|
+
select: mock(() => ({
|
|
596
|
+
from: mock(() => ({
|
|
597
|
+
where: mock(() => ({
|
|
598
|
+
limit: mock(() => Promise.resolve([existing])),
|
|
599
|
+
})),
|
|
600
|
+
})),
|
|
601
|
+
})),
|
|
602
|
+
insert: mock(() => ({
|
|
603
|
+
values: mock((values: Record<string, unknown>) => {
|
|
604
|
+
inserted = values;
|
|
605
|
+
return { onConflictDoUpdate: mock(() => Promise.resolve()) };
|
|
606
|
+
}),
|
|
607
|
+
})),
|
|
608
|
+
};
|
|
609
|
+
|
|
610
|
+
// The edited assertion (value 201) starts a NEW series in the same bucket.
|
|
611
|
+
await incrementHourlyAggregate({
|
|
612
|
+
db: db as never,
|
|
613
|
+
systemId: "sys-1",
|
|
614
|
+
configurationId: "config-1",
|
|
615
|
+
status: "healthy",
|
|
616
|
+
latencyMs: 100,
|
|
617
|
+
runTimestamp: new Date("2024-01-15T10:40:00Z"),
|
|
618
|
+
result: runResult([
|
|
619
|
+
{ ...outcome(true), key: OTHER_KEY, value: "201" },
|
|
620
|
+
]) as never,
|
|
621
|
+
collectorRegistry,
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
|
|
625
|
+
expect(aggregated.assertions).toEqual({
|
|
626
|
+
"uuid-1": {
|
|
627
|
+
[KEY]: { passCount: 4, failCount: 0 },
|
|
628
|
+
[OTHER_KEY]: { passCount: 1, failCount: 0 },
|
|
629
|
+
},
|
|
630
|
+
});
|
|
631
|
+
});
|
|
632
|
+
});
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import type { SafeDatabase, CollectorRegistry } from "@checkstack/backend-api";
|
|
2
|
+
import {
|
|
3
|
+
ASSERTIONS_AGG_KEY,
|
|
4
|
+
AssertionOutcomeSchema,
|
|
5
|
+
foldOutcomesIntoStats,
|
|
6
|
+
readAssertionStats,
|
|
7
|
+
type AssertionOutcome,
|
|
8
|
+
} from "@checkstack/healthcheck-common";
|
|
2
9
|
import { TDigest } from "tdigest";
|
|
3
10
|
import * as schema from "./schema";
|
|
4
11
|
import { healthCheckAggregates } from "./schema";
|
|
@@ -269,6 +276,31 @@ function mergeCollectorResults(params: {
|
|
|
269
276
|
return existingResult ?? undefined;
|
|
270
277
|
}
|
|
271
278
|
|
|
279
|
+
// Fold this run's structured assertion outcomes into the bucket's
|
|
280
|
+
// per-assertion pass/fail counts (a PLATFORM-owned top-level key, sibling
|
|
281
|
+
// of `collectors`, so collector mergers never see it). Folded for every
|
|
282
|
+
// entry that carries outcomes, even collectors without a mergeResult.
|
|
283
|
+
let assertionStats = readAssertionStats({
|
|
284
|
+
aggregatedResult: existingResult ?? undefined,
|
|
285
|
+
});
|
|
286
|
+
if (runCollectors) {
|
|
287
|
+
for (const [uuid, collectorData] of Object.entries(runCollectors)) {
|
|
288
|
+
const rawOutcomes = collectorData._assertions;
|
|
289
|
+
if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
|
|
290
|
+
const outcomes: AssertionOutcome[] = [];
|
|
291
|
+
for (const raw of rawOutcomes) {
|
|
292
|
+
const parsed = AssertionOutcomeSchema.safeParse(raw);
|
|
293
|
+
if (parsed.success) outcomes.push(parsed.data);
|
|
294
|
+
}
|
|
295
|
+
if (outcomes.length === 0) continue;
|
|
296
|
+
assertionStats = foldOutcomesIntoStats({
|
|
297
|
+
stats: assertionStats,
|
|
298
|
+
collectorEntryId: uuid,
|
|
299
|
+
outcomes,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
272
304
|
// Start with existing collectors or empty object
|
|
273
305
|
const existingCollectors = (existingResult?.collectors ?? {}) as Record<
|
|
274
306
|
string,
|
|
@@ -291,7 +323,7 @@ function mergeCollectorResults(params: {
|
|
|
291
323
|
const existingAggregate = existingCollectors[uuid];
|
|
292
324
|
|
|
293
325
|
// Strip internal fields from collector data for the run
|
|
294
|
-
const { _collectorId, _assertionFailed, ...collectorMetadata } =
|
|
326
|
+
const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
|
|
295
327
|
collectorData;
|
|
296
328
|
|
|
297
329
|
// Call the collector's mergeResult
|
|
@@ -313,5 +345,10 @@ function mergeCollectorResults(params: {
|
|
|
313
345
|
}
|
|
314
346
|
}
|
|
315
347
|
|
|
316
|
-
return {
|
|
348
|
+
return {
|
|
349
|
+
collectors: mergedCollectors,
|
|
350
|
+
...(assertionStats === undefined
|
|
351
|
+
? {}
|
|
352
|
+
: { [ASSERTIONS_AGG_KEY]: assertionStats }),
|
|
353
|
+
};
|
|
317
354
|
}
|
package/src/retention-job.ts
CHANGED
|
@@ -8,6 +8,12 @@ import {
|
|
|
8
8
|
DEFAULT_RETENTION_CONFIG,
|
|
9
9
|
} from "./schema";
|
|
10
10
|
import { eq, and, lt, sql, desc } from "drizzle-orm";
|
|
11
|
+
import {
|
|
12
|
+
ASSERTIONS_AGG_KEY,
|
|
13
|
+
mergeAssertionStats,
|
|
14
|
+
readAssertionStats,
|
|
15
|
+
type BucketAssertionStats,
|
|
16
|
+
} from "@checkstack/healthcheck-common";
|
|
11
17
|
import type { QueueManager } from "@checkstack/queue-api";
|
|
12
18
|
|
|
13
19
|
type Db = SafeDatabase<typeof schema>;
|
|
@@ -244,6 +250,8 @@ export interface HourlyAggregateRow {
|
|
|
244
250
|
minLatencyMs: number | null;
|
|
245
251
|
maxLatencyMs: number | null;
|
|
246
252
|
p95LatencyMs: number | null;
|
|
253
|
+
/** Carried for the per-assertion pass/fail counts (additive across hours). */
|
|
254
|
+
aggregatedResult?: Record<string, unknown> | null;
|
|
247
255
|
}
|
|
248
256
|
|
|
249
257
|
/** A computed daily aggregate ready to upsert. */
|
|
@@ -261,6 +269,12 @@ export interface DailyAggregateValues {
|
|
|
261
269
|
minLatencyMs: number | undefined;
|
|
262
270
|
maxLatencyMs: number | undefined;
|
|
263
271
|
p95LatencyMs: number | undefined;
|
|
272
|
+
/**
|
|
273
|
+
* Summed per-assertion pass/fail counts across the day's hourly buckets.
|
|
274
|
+
* The ONLY part of `aggregatedResult` that survives the daily rollup —
|
|
275
|
+
* assertion counts are purely additive, unlike strategy/collector states.
|
|
276
|
+
*/
|
|
277
|
+
assertionStats: BucketAssertionStats | undefined;
|
|
264
278
|
}
|
|
265
279
|
|
|
266
280
|
/**
|
|
@@ -296,6 +310,7 @@ export function buildDailyAggregates(
|
|
|
296
310
|
let degradedCount = 0;
|
|
297
311
|
let unhealthyCount = 0;
|
|
298
312
|
let latencySumMs = 0;
|
|
313
|
+
let assertionStats: BucketAssertionStats | undefined;
|
|
299
314
|
|
|
300
315
|
for (const a of rows) {
|
|
301
316
|
runCount += a.runCount;
|
|
@@ -308,6 +323,12 @@ export function buildDailyAggregates(
|
|
|
308
323
|
} else if (a.avgLatencyMs !== null) {
|
|
309
324
|
latencySumMs += a.avgLatencyMs * a.runCount;
|
|
310
325
|
}
|
|
326
|
+
assertionStats = mergeAssertionStats({
|
|
327
|
+
a: assertionStats,
|
|
328
|
+
b: readAssertionStats({
|
|
329
|
+
aggregatedResult: a.aggregatedResult ?? undefined,
|
|
330
|
+
}),
|
|
331
|
+
});
|
|
311
332
|
}
|
|
312
333
|
|
|
313
334
|
const minValues = rows
|
|
@@ -336,6 +357,7 @@ export function buildDailyAggregates(
|
|
|
336
357
|
maxLatencyMs: maxValues.length > 0 ? Math.max(...maxValues) : undefined,
|
|
337
358
|
// Use max of hourly p95s as an upper-bound approximation.
|
|
338
359
|
p95LatencyMs: p95Values.length > 0 ? Math.max(...p95Values) : undefined,
|
|
360
|
+
assertionStats,
|
|
339
361
|
});
|
|
340
362
|
}
|
|
341
363
|
|
|
@@ -370,6 +392,42 @@ async function rollupHourlyAggregates(params: RollupParams) {
|
|
|
370
392
|
// Fold into daily aggregates, preserving (day, environmentId, sourceId) series.
|
|
371
393
|
for (const daily of buildDailyAggregates(oldHourly)) {
|
|
372
394
|
const newLatencySum = daily.latencySumMs;
|
|
395
|
+
|
|
396
|
+
// Assertion pass/fail counts are the ONLY aggregatedResult content that
|
|
397
|
+
// survives the daily rollup (strategy/collector states cannot combine
|
|
398
|
+
// across hours). The conflict path merges in JS by pre-reading the
|
|
399
|
+
// existing daily row — safe because retention runs in a single work-queue
|
|
400
|
+
// consumer group, so there is no concurrent writer for this tuple.
|
|
401
|
+
let dailyAggregatedResult: Record<string, unknown> | undefined;
|
|
402
|
+
if (daily.assertionStats !== undefined) {
|
|
403
|
+
const [existingDaily] = await db
|
|
404
|
+
.select({ aggregatedResult: healthCheckAggregates.aggregatedResult })
|
|
405
|
+
.from(healthCheckAggregates)
|
|
406
|
+
.where(
|
|
407
|
+
and(
|
|
408
|
+
eq(healthCheckAggregates.systemId, systemId),
|
|
409
|
+
eq(healthCheckAggregates.configurationId, configurationId),
|
|
410
|
+
eq(healthCheckAggregates.bucketSize, "daily"),
|
|
411
|
+
eq(healthCheckAggregates.bucketStart, daily.bucketStart),
|
|
412
|
+
daily.environmentId === null
|
|
413
|
+
? sql`${healthCheckAggregates.environmentId} IS NULL`
|
|
414
|
+
: eq(healthCheckAggregates.environmentId, daily.environmentId),
|
|
415
|
+
daily.sourceId === null
|
|
416
|
+
? sql`${healthCheckAggregates.sourceId} IS NULL`
|
|
417
|
+
: eq(healthCheckAggregates.sourceId, daily.sourceId),
|
|
418
|
+
),
|
|
419
|
+
);
|
|
420
|
+
const mergedStats = mergeAssertionStats({
|
|
421
|
+
a: readAssertionStats({
|
|
422
|
+
aggregatedResult: existingDaily?.aggregatedResult ?? undefined,
|
|
423
|
+
}),
|
|
424
|
+
b: daily.assertionStats,
|
|
425
|
+
});
|
|
426
|
+
if (mergedStats !== undefined) {
|
|
427
|
+
dailyAggregatedResult = { [ASSERTIONS_AGG_KEY]: mergedStats };
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
373
431
|
// Upsert the daily aggregate. A row may already exist for this
|
|
374
432
|
// (configurationId, systemId, environmentId, day, daily, sourceId) tuple if
|
|
375
433
|
// a prior rollup ran and then late-arriving hourly buckets (e.g. from a
|
|
@@ -394,7 +452,8 @@ async function rollupHourlyAggregates(params: RollupParams) {
|
|
|
394
452
|
minLatencyMs: daily.minLatencyMs,
|
|
395
453
|
maxLatencyMs: daily.maxLatencyMs,
|
|
396
454
|
p95LatencyMs: daily.p95LatencyMs,
|
|
397
|
-
|
|
455
|
+
// Only the additive assertion counts survive across hours.
|
|
456
|
+
aggregatedResult: dailyAggregatedResult,
|
|
398
457
|
})
|
|
399
458
|
.onConflictDoUpdate({
|
|
400
459
|
target: [...DAILY_AGGREGATE_CONFLICT_TARGET],
|
|
@@ -417,6 +476,10 @@ async function rollupHourlyAggregates(params: RollupParams) {
|
|
|
417
476
|
daily.p95LatencyMs === undefined
|
|
418
477
|
? sql`${healthCheckAggregates.p95LatencyMs}`
|
|
419
478
|
: sql`GREATEST(COALESCE(${healthCheckAggregates.p95LatencyMs}, ${daily.p95LatencyMs}), ${daily.p95LatencyMs})`,
|
|
479
|
+
// JS-merged above (existing row's counts + this rollup's counts).
|
|
480
|
+
...(dailyAggregatedResult === undefined
|
|
481
|
+
? {}
|
|
482
|
+
: { aggregatedResult: dailyAggregatedResult }),
|
|
420
483
|
},
|
|
421
484
|
});
|
|
422
485
|
}
|