@checkstack/healthcheck-backend 1.14.0 → 1.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1459,3 +1459,185 @@ describe("recomputeSystemRollupHealth", () => {
1459
1459
  expect(logger.error).toHaveBeenCalledTimes(1);
1460
1460
  });
1461
1461
  });
1462
+
1463
+ describe("executeHealthCheckJob - structured assertion outcomes", () => {
1464
+ it("stores _assertions per collector entry and downgrades on failure", async () => {
1465
+ const mockDb = createMockDb();
1466
+ const mockRegistry = createMockRegistry();
1467
+ const mockLogger = createMockLogger();
1468
+ const mockQueueManager = createMockQueueManager();
1469
+ const mockCatalogClient = createMockCatalogClient();
1470
+ const mockMaintenanceClient = createMockMaintenanceClient();
1471
+ const mockIncidentClient = createMockIncidentClient();
1472
+ const mockSignalService = createMockSignalService();
1473
+
1474
+ (mockCatalogClient.getSystem as any) = mock(async () => ({
1475
+ id: "system-1",
1476
+ name: "web-01",
1477
+ }));
1478
+
1479
+ // One collector entry with two assertions: isTrue passes, equals fails.
1480
+ let selectCallCount = 0;
1481
+ (mockDb.select as any) = mock(() => {
1482
+ selectCallCount++;
1483
+ if (selectCallCount === 2) {
1484
+ return {
1485
+ from: mock(() => ({
1486
+ innerJoin: mock(() => ({
1487
+ where: mock(() =>
1488
+ Promise.resolve([
1489
+ {
1490
+ configId: "config-1",
1491
+ configName: "checkout",
1492
+ strategyId: "test-strategy",
1493
+ config: { timeout: 5000 },
1494
+ collectors: [
1495
+ {
1496
+ id: "col-1",
1497
+ collectorId: "test-collector",
1498
+ config: {},
1499
+ assertions: [
1500
+ { field: "ok", operator: "isTrue" },
1501
+ { field: "code", operator: "equals", value: 200 },
1502
+ ],
1503
+ },
1504
+ ],
1505
+ interval: 45,
1506
+ enabled: true,
1507
+ paused: false,
1508
+ includeLocal: true,
1509
+ satelliteIds: [],
1510
+ },
1511
+ ]),
1512
+ ),
1513
+ })),
1514
+ })),
1515
+ };
1516
+ }
1517
+ return {
1518
+ from: mock(() => ({
1519
+ innerJoin: mock(() => ({
1520
+ where: mock(() => Promise.resolve([])),
1521
+ })),
1522
+ })),
1523
+ };
1524
+ });
1525
+
1526
+ // Capture every insert's values so we can find the stored run.
1527
+ const insertedValues: Record<string, unknown>[] = [];
1528
+ (mockDb.insert as any) = mock(() => ({
1529
+ values: mock((vals: Record<string, unknown>) => {
1530
+ insertedValues.push(vals);
1531
+ return Object.assign(Promise.resolve(), {
1532
+ onConflictDoUpdate: mock(() => Promise.resolve()),
1533
+ onConflictDoNothing: mock(() => Promise.resolve()),
1534
+ returning: mock(() => Promise.resolve([])),
1535
+ });
1536
+ }),
1537
+ }));
1538
+
1539
+ const mockCollectorRegistry = {
1540
+ register: mock(() => {}),
1541
+ getCollector: mock(() => ({
1542
+ collector: {
1543
+ id: "test-collector",
1544
+ execute: mock(async () => ({ result: { ok: true, code: 404 } })),
1545
+ config: new Versioned({ version: 1, schema: z.object({}) }),
1546
+ result: new Versioned({
1547
+ version: 1,
1548
+ schema: z.object({ ok: z.boolean(), code: z.number() }),
1549
+ }),
1550
+ mergeResult: mock(() => ({})),
1551
+ },
1552
+ })),
1553
+ getCollectors: mock(() => []),
1554
+ };
1555
+
1556
+ const queue =
1557
+ mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
1558
+ let capturedHandler:
1559
+ | ((job: { data: HealthCheckJobPayload }) => Promise<void>)
1560
+ | undefined;
1561
+ (queue.consume as any) = mock(
1562
+ async (
1563
+ handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
1564
+ ) => {
1565
+ capturedHandler = handler;
1566
+ },
1567
+ );
1568
+
1569
+ await setupHealthCheckWorker({
1570
+ db: mockDb as unknown as Parameters<
1571
+ typeof setupHealthCheckWorker
1572
+ >[0]["db"],
1573
+ advisoryLock: mockAdvisoryLock,
1574
+ registry: mockRegistry,
1575
+ collectorRegistry: mockCollectorRegistry as unknown as Parameters<
1576
+ typeof setupHealthCheckWorker
1577
+ >[0]["collectorRegistry"],
1578
+ logger: mockLogger,
1579
+ queueManager: mockQueueManager,
1580
+ signalService: mockSignalService,
1581
+ catalogClient: mockCatalogClient as unknown as Parameters<
1582
+ typeof setupHealthCheckWorker
1583
+ >[0]["catalogClient"],
1584
+ notificationClient: {
1585
+ notifyForSubscription: () => Promise.resolve({ notifiedCount: 0 }),
1586
+ } as unknown as Parameters<
1587
+ typeof setupHealthCheckWorker
1588
+ >[0]["notificationClient"],
1589
+ maintenanceClient: mockMaintenanceClient as unknown as Parameters<
1590
+ typeof setupHealthCheckWorker
1591
+ >[0]["maintenanceClient"],
1592
+ incidentClient: mockIncidentClient as unknown as Parameters<
1593
+ typeof setupHealthCheckWorker
1594
+ >[0]["incidentClient"],
1595
+ getEmitHook: () => undefined,
1596
+ cache: passthroughCache,
1597
+ });
1598
+
1599
+ if (capturedHandler) {
1600
+ // Downstream aggregation touches DB surfaces the lightweight mock
1601
+ // doesn't model; the run insert we assert on happens before that.
1602
+ await capturedHandler({
1603
+ data: { configId: "config-1", systemId: "system-1" },
1604
+ }).catch(() => {});
1605
+ }
1606
+
1607
+ const runInsert = insertedValues.find(
1608
+ (vals) => "status" in vals && "result" in vals,
1609
+ );
1610
+ expect(runInsert).toBeDefined();
1611
+ // The failed `code equals 200` assertion downgrades the run.
1612
+ expect(runInsert?.status).toBe("unhealthy");
1613
+
1614
+ const runResult = runInsert?.result as {
1615
+ metadata: {
1616
+ collectors: Record<
1617
+ string,
1618
+ {
1619
+ _assertionFailed?: string;
1620
+ _assertions?: {
1621
+ field: string;
1622
+ passed: boolean;
1623
+ actual?: string;
1624
+ }[];
1625
+ }
1626
+ >;
1627
+ };
1628
+ };
1629
+ const entry = runResult.metadata.collectors["col-1"];
1630
+ expect(entry._assertionFailed).toBe("code equals 200");
1631
+ expect(entry._assertions?.length).toBe(2);
1632
+ expect(entry._assertions?.[0]).toMatchObject({
1633
+ field: "ok",
1634
+ passed: true,
1635
+ actual: "true",
1636
+ });
1637
+ expect(entry._assertions?.[1]).toMatchObject({
1638
+ field: "code",
1639
+ passed: false,
1640
+ actual: "404",
1641
+ });
1642
+ });
1643
+ });
@@ -46,7 +46,11 @@ import { NotificationApi } from "@checkstack/notification-common";
46
46
  import { healthcheckSystemSubscription } from "@checkstack/healthcheck-common";
47
47
  import { resolveRoute, type InferClient, extractErrorMessage} from "@checkstack/common";
48
48
  import { secretEnvMappingSchema } from "@checkstack/secrets-common";
49
- import type { SecretResolverService } from "@checkstack/secrets-backend";
49
+ import type {
50
+ SecretResolverService,
51
+ InternalSecretsService,
52
+ } from "@checkstack/secrets-backend";
53
+ import { inflateConfigSecrets } from "./config-secrets";
50
54
  import { HealthCheckService } from "./service";
51
55
  import { healthCheckHooks } from "./hooks";
52
56
  import { incrementHourlyAggregate } from "./realtime-aggregation";
@@ -56,7 +60,8 @@ import {
56
60
  shouldNotifyTransition,
57
61
  } from "./notification-policy";
58
62
  import { recordStateTransition } from "./state-transitions";
59
- import { evaluateCollectorAssertions } from "./collector-assertions";
63
+ import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
64
+ import type { AssertionOutcome } from "@checkstack/healthcheck-common";
60
65
  import {
61
66
  writeHealthEntity,
62
67
  createHealthEntitySerializer,
@@ -538,6 +543,14 @@ async function executeHealthCheckJob(props: {
538
543
  * / test isolation.
539
544
  */
540
545
  secretResolver?: SecretResolverService;
546
+ /**
547
+ * Internal secret store. When set (together with `secretResolver`), stored
548
+ * strategy/collector config `x-secret` fields - internal markers and
549
+ * `${{ secrets.* }}` references - are INFLATED to their real values just
550
+ * before use, in memory only. Optional for version-skew / test isolation;
551
+ * without it, marker-bearing configs fail their runs clearly.
552
+ */
553
+ internalSecrets?: InternalSecretsService;
541
554
  }): Promise<void> {
542
555
  const {
543
556
  payload,
@@ -555,6 +568,7 @@ async function executeHealthCheckJob(props: {
555
568
  getHealthEntity,
556
569
  cache,
557
570
  secretResolver,
571
+ internalSecrets,
558
572
  } = props;
559
573
  const { configId, systemId } = payload;
560
574
 
@@ -656,6 +670,22 @@ async function executeHealthCheckJob(props: {
656
670
  return;
657
671
  }
658
672
 
673
+ // Inflate stored secret markers / `${{ secrets.* }}` references to their
674
+ // real values ONCE, memory-only, BEFORE migrate+validate - so validation
675
+ // sees real values. Old-shape rows (whose current-schema secret keys do
676
+ // not exist yet) and legacy bare literals pass through untouched.
677
+ let rawStrategyConfig = configRow.config;
678
+ if (internalSecrets && secretResolver) {
679
+ const inflated = await inflateConfigSecrets({
680
+ configurationId: configId,
681
+ scope: { kind: "strategy" },
682
+ schema: strategy.config.schema,
683
+ config: configRow.config,
684
+ deps: { internalSecrets, secretResolver },
685
+ });
686
+ rawStrategyConfig = inflated.config;
687
+ }
688
+
659
689
  // Migrate the stored (UNVERSIONED) strategy config ONCE, before the
660
690
  // per-environment render loop, so every env renders from the same
661
691
  // migrated shape. Stored configs predate explicit versioning and may be
@@ -663,7 +693,7 @@ async function executeHealthCheckJob(props: {
663
693
  // -on-read runs the declared migration chain, then validates. The
664
694
  // migrations are idempotent, so an already-current config is a no-op.
665
695
  const strategyConfig: BaseStrategyConfig =
666
- await strategy.config.parseAssumingV1(configRow.config);
696
+ await strategy.config.parseAssumingV1(rawStrategyConfig);
667
697
  const executionTimeout = strategyConfig.timeout ?? 60_000;
668
698
 
669
699
  // ── Per-environment fan-out (§7) ────────────────────────────────────────
@@ -872,9 +902,26 @@ async function executeHealthCheckJob(props: {
872
902
  // reads the raw `secretEnv` mapping (a constant string field
873
903
  // unaffected by the strategy/collector reshapes), keeping the
874
904
  // migrate -> secret resolve -> render -> execute order intact.
905
+ // Inflate this entry's secret markers / references (memory
906
+ // only) before its migrate+validate parse, mirroring the
907
+ // strategy-config inflation above.
908
+ let rawCollectorConfig = collectorEntry.config;
909
+ if (internalSecrets && secretResolver) {
910
+ const inflated = await inflateConfigSecrets({
911
+ configurationId: configId,
912
+ scope: {
913
+ kind: "collector",
914
+ entryId: collectorEntry.id,
915
+ },
916
+ schema: registered.collector.config.schema,
917
+ config: collectorEntry.config,
918
+ deps: { internalSecrets, secretResolver },
919
+ });
920
+ rawCollectorConfig = inflated.config;
921
+ }
875
922
  const migratedCollectorConfig =
876
923
  await registered.collector.config.parseAssumingV1(
877
- collectorEntry.config,
924
+ rawCollectorConfig,
878
925
  );
879
926
 
880
927
  // (2) Environment/templating pass for the collector config -
@@ -901,13 +948,18 @@ async function executeHealthCheckJob(props: {
901
948
  collectorError = collectorResult.error;
902
949
  }
903
950
 
904
- // Evaluate per-collector assertions (plain fields + JSONPath)
951
+ // Evaluate per-collector assertions (plain fields + JSONPath).
952
+ // ALL outcomes are stored (pass AND fail) so assertions are
953
+ // analyzable over time, not only visible on failure.
905
954
  let assertionFailed: string | undefined;
955
+ let assertionOutcomes: AssertionOutcome[] = [];
906
956
  if (collectorResult.result) {
907
- assertionFailed = evaluateCollectorAssertions({
957
+ const evaluation = evaluateCollectorAssertionOutcomes({
908
958
  assertions: collectorEntry.assertions,
909
959
  result: collectorResult.result as Record<string, unknown>,
910
960
  });
961
+ assertionFailed = evaluation.firstFailureMessage;
962
+ assertionOutcomes = evaluation.outcomes;
911
963
  if (assertionFailed) {
912
964
  logger.debug(
913
965
  `Collector ${storageKey} assertion failed: ${assertionFailed}`,
@@ -931,6 +983,9 @@ async function executeHealthCheckJob(props: {
931
983
  _collectorId: collectorEntry.collectorId,
932
984
  _assertionFailed: assertionFailed,
933
985
  _collectorError: collectorError,
986
+ ...(assertionOutcomes.length > 0
987
+ ? { _assertions: assertionOutcomes }
988
+ : {}),
934
989
  ...strippedResult,
935
990
  },
936
991
  };
@@ -1518,6 +1573,7 @@ export async function setupHealthCheckWorker(props: {
1518
1573
  getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
1519
1574
  cache: HealthCheckCache;
1520
1575
  secretResolver?: SecretResolverService;
1576
+ internalSecrets?: InternalSecretsService;
1521
1577
  }): Promise<void> {
1522
1578
  const {
1523
1579
  db,
@@ -1535,6 +1591,7 @@ export async function setupHealthCheckWorker(props: {
1535
1591
  getHealthEntity,
1536
1592
  cache,
1537
1593
  secretResolver,
1594
+ internalSecrets,
1538
1595
  } = props;
1539
1596
 
1540
1597
  const queue =
@@ -1559,6 +1616,7 @@ export async function setupHealthCheckWorker(props: {
1559
1616
  getHealthEntity,
1560
1617
  cache,
1561
1618
  secretResolver,
1619
+ internalSecrets,
1562
1620
  });
1563
1621
  },
1564
1622
  {
@@ -5,6 +5,7 @@ import {
5
5
  deserializeTDigest,
6
6
  incrementHourlyAggregate,
7
7
  } from "./realtime-aggregation";
8
+ import { computeAssertionKey } from "@checkstack/healthcheck-common";
8
9
  import { TDigest } from "tdigest";
9
10
 
10
11
  describe("getHourBucketStart", () => {
@@ -494,3 +495,138 @@ describe("incrementHourlyAggregate", () => {
494
495
  expect(inserted.maxLatencyMs).toBe(150);
495
496
  });
496
497
  });
498
+
499
+ describe("incrementHourlyAggregate - assertion stats folding", () => {
500
+ const KEY = computeAssertionKey({
501
+ assertion: { field: "statusCode", operator: "equals", value: 200 },
502
+ });
503
+
504
+ const outcome = (passed: boolean) => ({
505
+ key: KEY,
506
+ field: "statusCode",
507
+ operator: "equals",
508
+ value: "200",
509
+ passed,
510
+ });
511
+
512
+ const collectorRegistry = {
513
+ register: mock(() => {}),
514
+ getCollector: mock(() => ({
515
+ collector: {
516
+ id: "test-collector",
517
+ mergeResult: mock(() => ({ count: 1 })),
518
+ },
519
+ })),
520
+ getCollectors: mock(() => []),
521
+ } as unknown as Parameters<
522
+ typeof incrementHourlyAggregate
523
+ >[0]["collectorRegistry"];
524
+
525
+ function runResult(outcomes: unknown[]) {
526
+ return {
527
+ metadata: {
528
+ collectors: {
529
+ "uuid-1": {
530
+ _collectorId: "test-collector",
531
+ _assertions: outcomes,
532
+ statusCode: 200,
533
+ },
534
+ },
535
+ },
536
+ };
537
+ }
538
+
539
+ it("folds a run's outcomes into the bucket's per-assertion counts", async () => {
540
+ let inserted: Record<string, unknown> | undefined;
541
+ const db = {
542
+ select: mock(() => ({
543
+ from: mock(() => ({
544
+ where: mock(() => ({
545
+ limit: mock(() => Promise.resolve([])),
546
+ })),
547
+ })),
548
+ })),
549
+ insert: mock(() => ({
550
+ values: mock((values: Record<string, unknown>) => {
551
+ inserted = values;
552
+ return { onConflictDoUpdate: mock(() => Promise.resolve()) };
553
+ }),
554
+ })),
555
+ };
556
+
557
+ await incrementHourlyAggregate({
558
+ db: db as never,
559
+ systemId: "sys-1",
560
+ configurationId: "config-1",
561
+ status: "unhealthy",
562
+ latencyMs: 100,
563
+ runTimestamp: new Date("2024-01-15T10:35:00Z"),
564
+ result: runResult([outcome(true), outcome(false)]) as never,
565
+ collectorRegistry,
566
+ });
567
+
568
+ const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
569
+ expect(aggregated.assertions).toEqual({
570
+ "uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
571
+ });
572
+ // The internal _assertions never leak into collector merge output.
573
+ const collectors = aggregated.collectors as Record<
574
+ string,
575
+ Record<string, unknown>
576
+ >;
577
+ expect(collectors["uuid-1"]._assertions).toBeUndefined();
578
+ });
579
+
580
+ it("increments existing counts and keeps distinct keys separate (mid-bucket config edit)", async () => {
581
+ const OTHER_KEY = computeAssertionKey({
582
+ assertion: { field: "statusCode", operator: "equals", value: 201 },
583
+ });
584
+ const existing = {
585
+ tdigestState: null,
586
+ minLatencyMs: null,
587
+ maxLatencyMs: null,
588
+ aggregatedResult: {
589
+ collectors: {},
590
+ assertions: { "uuid-1": { [KEY]: { passCount: 4, failCount: 0 } } },
591
+ },
592
+ };
593
+ let inserted: Record<string, unknown> | undefined;
594
+ const db = {
595
+ select: mock(() => ({
596
+ from: mock(() => ({
597
+ where: mock(() => ({
598
+ limit: mock(() => Promise.resolve([existing])),
599
+ })),
600
+ })),
601
+ })),
602
+ insert: mock(() => ({
603
+ values: mock((values: Record<string, unknown>) => {
604
+ inserted = values;
605
+ return { onConflictDoUpdate: mock(() => Promise.resolve()) };
606
+ }),
607
+ })),
608
+ };
609
+
610
+ // The edited assertion (value 201) starts a NEW series in the same bucket.
611
+ await incrementHourlyAggregate({
612
+ db: db as never,
613
+ systemId: "sys-1",
614
+ configurationId: "config-1",
615
+ status: "healthy",
616
+ latencyMs: 100,
617
+ runTimestamp: new Date("2024-01-15T10:40:00Z"),
618
+ result: runResult([
619
+ { ...outcome(true), key: OTHER_KEY, value: "201" },
620
+ ]) as never,
621
+ collectorRegistry,
622
+ });
623
+
624
+ const aggregated = inserted?.aggregatedResult as Record<string, unknown>;
625
+ expect(aggregated.assertions).toEqual({
626
+ "uuid-1": {
627
+ [KEY]: { passCount: 4, failCount: 0 },
628
+ [OTHER_KEY]: { passCount: 1, failCount: 0 },
629
+ },
630
+ });
631
+ });
632
+ });
@@ -1,4 +1,11 @@
1
1
  import type { SafeDatabase, CollectorRegistry } from "@checkstack/backend-api";
2
+ import {
3
+ ASSERTIONS_AGG_KEY,
4
+ AssertionOutcomeSchema,
5
+ foldOutcomesIntoStats,
6
+ readAssertionStats,
7
+ type AssertionOutcome,
8
+ } from "@checkstack/healthcheck-common";
2
9
  import { TDigest } from "tdigest";
3
10
  import * as schema from "./schema";
4
11
  import { healthCheckAggregates } from "./schema";
@@ -269,6 +276,31 @@ function mergeCollectorResults(params: {
269
276
  return existingResult ?? undefined;
270
277
  }
271
278
 
279
+ // Fold this run's structured assertion outcomes into the bucket's
280
+ // per-assertion pass/fail counts (a PLATFORM-owned top-level key, sibling
281
+ // of `collectors`, so collector mergers never see it). Folded for every
282
+ // entry that carries outcomes, even collectors without a mergeResult.
283
+ let assertionStats = readAssertionStats({
284
+ aggregatedResult: existingResult ?? undefined,
285
+ });
286
+ if (runCollectors) {
287
+ for (const [uuid, collectorData] of Object.entries(runCollectors)) {
288
+ const rawOutcomes = collectorData._assertions;
289
+ if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
290
+ const outcomes: AssertionOutcome[] = [];
291
+ for (const raw of rawOutcomes) {
292
+ const parsed = AssertionOutcomeSchema.safeParse(raw);
293
+ if (parsed.success) outcomes.push(parsed.data);
294
+ }
295
+ if (outcomes.length === 0) continue;
296
+ assertionStats = foldOutcomesIntoStats({
297
+ stats: assertionStats,
298
+ collectorEntryId: uuid,
299
+ outcomes,
300
+ });
301
+ }
302
+ }
303
+
272
304
  // Start with existing collectors or empty object
273
305
  const existingCollectors = (existingResult?.collectors ?? {}) as Record<
274
306
  string,
@@ -291,7 +323,7 @@ function mergeCollectorResults(params: {
291
323
  const existingAggregate = existingCollectors[uuid];
292
324
 
293
325
  // Strip internal fields from collector data for the run
294
- const { _collectorId, _assertionFailed, ...collectorMetadata } =
326
+ const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
295
327
  collectorData;
296
328
 
297
329
  // Call the collector's mergeResult
@@ -313,5 +345,10 @@ function mergeCollectorResults(params: {
313
345
  }
314
346
  }
315
347
 
316
- return { collectors: mergedCollectors };
348
+ return {
349
+ collectors: mergedCollectors,
350
+ ...(assertionStats === undefined
351
+ ? {}
352
+ : { [ASSERTIONS_AGG_KEY]: assertionStats }),
353
+ };
317
354
  }
@@ -8,6 +8,12 @@ import {
8
8
  DEFAULT_RETENTION_CONFIG,
9
9
  } from "./schema";
10
10
  import { eq, and, lt, sql, desc } from "drizzle-orm";
11
+ import {
12
+ ASSERTIONS_AGG_KEY,
13
+ mergeAssertionStats,
14
+ readAssertionStats,
15
+ type BucketAssertionStats,
16
+ } from "@checkstack/healthcheck-common";
11
17
  import type { QueueManager } from "@checkstack/queue-api";
12
18
 
13
19
  type Db = SafeDatabase<typeof schema>;
@@ -244,6 +250,8 @@ export interface HourlyAggregateRow {
244
250
  minLatencyMs: number | null;
245
251
  maxLatencyMs: number | null;
246
252
  p95LatencyMs: number | null;
253
+ /** Carried for the per-assertion pass/fail counts (additive across hours). */
254
+ aggregatedResult?: Record<string, unknown> | null;
247
255
  }
248
256
 
249
257
  /** A computed daily aggregate ready to upsert. */
@@ -261,6 +269,12 @@ export interface DailyAggregateValues {
261
269
  minLatencyMs: number | undefined;
262
270
  maxLatencyMs: number | undefined;
263
271
  p95LatencyMs: number | undefined;
272
+ /**
273
+ * Summed per-assertion pass/fail counts across the day's hourly buckets.
274
+ * The ONLY part of `aggregatedResult` that survives the daily rollup —
275
+ * assertion counts are purely additive, unlike strategy/collector states.
276
+ */
277
+ assertionStats: BucketAssertionStats | undefined;
264
278
  }
265
279
 
266
280
  /**
@@ -296,6 +310,7 @@ export function buildDailyAggregates(
296
310
  let degradedCount = 0;
297
311
  let unhealthyCount = 0;
298
312
  let latencySumMs = 0;
313
+ let assertionStats: BucketAssertionStats | undefined;
299
314
 
300
315
  for (const a of rows) {
301
316
  runCount += a.runCount;
@@ -308,6 +323,12 @@ export function buildDailyAggregates(
308
323
  } else if (a.avgLatencyMs !== null) {
309
324
  latencySumMs += a.avgLatencyMs * a.runCount;
310
325
  }
326
+ assertionStats = mergeAssertionStats({
327
+ a: assertionStats,
328
+ b: readAssertionStats({
329
+ aggregatedResult: a.aggregatedResult ?? undefined,
330
+ }),
331
+ });
311
332
  }
312
333
 
313
334
  const minValues = rows
@@ -336,6 +357,7 @@ export function buildDailyAggregates(
336
357
  maxLatencyMs: maxValues.length > 0 ? Math.max(...maxValues) : undefined,
337
358
  // Use max of hourly p95s as an upper-bound approximation.
338
359
  p95LatencyMs: p95Values.length > 0 ? Math.max(...p95Values) : undefined,
360
+ assertionStats,
339
361
  });
340
362
  }
341
363
 
@@ -370,6 +392,42 @@ async function rollupHourlyAggregates(params: RollupParams) {
370
392
  // Fold into daily aggregates, preserving (day, environmentId, sourceId) series.
371
393
  for (const daily of buildDailyAggregates(oldHourly)) {
372
394
  const newLatencySum = daily.latencySumMs;
395
+
396
+ // Assertion pass/fail counts are the ONLY aggregatedResult content that
397
+ // survives the daily rollup (strategy/collector states cannot combine
398
+ // across hours). The conflict path merges in JS by pre-reading the
399
+ // existing daily row — safe because retention runs in a single work-queue
400
+ // consumer group, so there is no concurrent writer for this tuple.
401
+ let dailyAggregatedResult: Record<string, unknown> | undefined;
402
+ if (daily.assertionStats !== undefined) {
403
+ const [existingDaily] = await db
404
+ .select({ aggregatedResult: healthCheckAggregates.aggregatedResult })
405
+ .from(healthCheckAggregates)
406
+ .where(
407
+ and(
408
+ eq(healthCheckAggregates.systemId, systemId),
409
+ eq(healthCheckAggregates.configurationId, configurationId),
410
+ eq(healthCheckAggregates.bucketSize, "daily"),
411
+ eq(healthCheckAggregates.bucketStart, daily.bucketStart),
412
+ daily.environmentId === null
413
+ ? sql`${healthCheckAggregates.environmentId} IS NULL`
414
+ : eq(healthCheckAggregates.environmentId, daily.environmentId),
415
+ daily.sourceId === null
416
+ ? sql`${healthCheckAggregates.sourceId} IS NULL`
417
+ : eq(healthCheckAggregates.sourceId, daily.sourceId),
418
+ ),
419
+ );
420
+ const mergedStats = mergeAssertionStats({
421
+ a: readAssertionStats({
422
+ aggregatedResult: existingDaily?.aggregatedResult ?? undefined,
423
+ }),
424
+ b: daily.assertionStats,
425
+ });
426
+ if (mergedStats !== undefined) {
427
+ dailyAggregatedResult = { [ASSERTIONS_AGG_KEY]: mergedStats };
428
+ }
429
+ }
430
+
373
431
  // Upsert the daily aggregate. A row may already exist for this
374
432
  // (configurationId, systemId, environmentId, day, daily, sourceId) tuple if
375
433
  // a prior rollup ran and then late-arriving hourly buckets (e.g. from a
@@ -394,7 +452,8 @@ async function rollupHourlyAggregates(params: RollupParams) {
394
452
  minLatencyMs: daily.minLatencyMs,
395
453
  maxLatencyMs: daily.maxLatencyMs,
396
454
  p95LatencyMs: daily.p95LatencyMs,
397
- aggregatedResult: undefined, // Cannot combine result across hours
455
+ // Only the additive assertion counts survive across hours.
456
+ aggregatedResult: dailyAggregatedResult,
398
457
  })
399
458
  .onConflictDoUpdate({
400
459
  target: [...DAILY_AGGREGATE_CONFLICT_TARGET],
@@ -417,6 +476,10 @@ async function rollupHourlyAggregates(params: RollupParams) {
417
476
  daily.p95LatencyMs === undefined
418
477
  ? sql`${healthCheckAggregates.p95LatencyMs}`
419
478
  : sql`GREATEST(COALESCE(${healthCheckAggregates.p95LatencyMs}, ${daily.p95LatencyMs}), ${daily.p95LatencyMs})`,
479
+ // JS-merged above (existing row's counts + this rollup's counts).
480
+ ...(dailyAggregatedResult === undefined
481
+ ? {}
482
+ : { aggregatedResult: dailyAggregatedResult }),
420
483
  },
421
484
  });
422
485
  }