@checkstack/healthcheck-backend 1.15.0 → 1.16.0

package/CHANGELOG.md

@@ -1,5 +1,82 @@
 # @checkstack/healthcheck-backend
 
+## 1.16.0
+
+### Minor Changes
+
+- c55d7c6: Make collector assertions analyzable: structured per-assertion outcomes on
+  every run, pass/fail counts in every aggregate tier, and dedicated analysis
+  surfaces. Previously a passing assertion left no trace and only the first
+  failure was recorded as a string.
+
+  - `@checkstack/healthcheck-common` adds the assertion-analytics contract:
+    `AssertionOutcomeSchema`, per-bucket `BucketAssertionStats` (stored under
+    the platform-owned top-level `assertions` key of `aggregatedResult`), and
+    the canonical assertion identity key (`computeAssertionKey` /
+    `parseAssertionKey`, a JSON tuple of field/jsonPath/operator/value).
+    Editing an assertion starts a new series; identical duplicates collapse.
+  - The executor evaluates ALL assertions (no first-failure short-circuit) and
+    stores `_assertions` on each collector entry alongside the unchanged
+    `_assertionFailed` compatibility string. Pass/fail counts are folded into
+    the hourly realtime aggregation, the on-read raw tier, cross-tier bucket
+    re-merges, and the daily retention rollup (assertion counts are the only
+    `aggregatedResult` content that survives the rollup - they are purely
+    additive), so assertion analytics do not silently end at the hourly
+    retention horizon.
+  - Satellite ingest now evaluates assertions on the core
+    (`ingestSatelliteResult`), downgrading a satellite-reported healthy run
+    whose assertions fail, and strips ephemeral result fields (e.g. raw HTTP
+    bodies) at ingest for parity with local runs. BEHAVIOR CHANGE:
+    satellite-executed checks previously never enforced assertions at all;
+    they now do, with no satellite upgrade or wire-protocol change. Buffered
+    satellite results are evaluated against the configuration current at
+    ingest time.
+  - The run detail gains an Assertions tab (per-collector groups, pass AND
+    fail rows with expected vs actual, a legacy fallback for pre-feature
+    runs), and the drawer's auto-chart grid leads each collector group with
+    per-assertion pass-rate tiles (sparkline of per-bucket pass rate,
+    expandable to a pass/fail StackedTimeline; currently-configured assertions
+    appear before any data exists, historical-only series are flagged).
+
+  State & scale: all new state lives in the existing `healthCheckRuns.result`
+  and `healthCheckAggregates.aggregated_result` jsonb columns (durable, shared
+  Postgres - no new tables, no pod-local state); reads resolve identically on
+  every pod; the run-vs-bucket duplication is the platform's existing
+  raw-vs-aggregate tiering with the existing single-writer upsert paths.
+
+### Patch Changes
+
+- Updated dependencies [c55d7c6]
+- Updated dependencies [c55d7c6]
+- Updated dependencies [a83bcc2]
+- Updated dependencies [c55d7c6]
+  - @checkstack/ai-backend@0.10.7
+  - @checkstack/healthcheck-common@1.13.0
+  - @checkstack/common@0.21.0
+  - @checkstack/automation-backend@0.10.9
+  - @checkstack/catalog-backend@1.6.7
+  - @checkstack/incident-backend@1.9.5
+  - @checkstack/satellite-backend@0.8.1
+  - @checkstack/backend-api@0.29.1
+  - @checkstack/sdk@0.123.1
+  - @checkstack/ai-common@0.6.5
+  - @checkstack/cache-api@0.3.18
+  - @checkstack/catalog-common@2.6.2
+  - @checkstack/command-backend@0.2.19
+  - @checkstack/gitops-backend@0.5.19
+  - @checkstack/gitops-common@0.7.2
+  - @checkstack/incident-common@1.7.2
+  - @checkstack/maintenance-common@1.8.2
+  - @checkstack/notification-common@1.5.2
+  - @checkstack/queue-api@0.3.18
+  - @checkstack/script-packages-backend@0.3.23
+  - @checkstack/secrets-backend@0.3.1
+  - @checkstack/secrets-common@0.3.1
+  - @checkstack/signal-common@0.2.16
+  - @checkstack/status-page-backend@0.4.6
+  - @checkstack/status-page-common@0.5.2
+  - @checkstack/cache-utils@0.2.23
+
 ## 1.15.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.15.0",
+  "version": "1.16.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,32 +14,32 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/ai-backend": "0.10.6",
-    "@checkstack/ai-common": "0.6.4",
-    "@checkstack/automation-backend": "0.10.8",
-    "@checkstack/backend-api": "0.29.0",
-    "@checkstack/cache-api": "0.3.17",
-    "@checkstack/cache-utils": "0.2.22",
-    "@checkstack/catalog-backend": "1.6.6",
-    "@checkstack/catalog-common": "2.6.1",
-    "@checkstack/command-backend": "0.2.18",
-    "@checkstack/common": "0.20.0",
-    "@checkstack/gitops-backend": "0.5.18",
-    "@checkstack/gitops-common": "0.7.1",
-    "@checkstack/healthcheck-common": "1.12.0",
-    "@checkstack/incident-backend": "1.9.4",
-    "@checkstack/incident-common": "1.7.1",
-    "@checkstack/maintenance-common": "1.8.1",
-    "@checkstack/notification-common": "1.5.1",
-    "@checkstack/queue-api": "0.3.17",
-    "@checkstack/satellite-backend": "0.8.0",
-    "@checkstack/script-packages-backend": "0.3.22",
-    "@checkstack/sdk": "0.122.1",
-    "@checkstack/secrets-backend": "0.3.0",
-    "@checkstack/secrets-common": "0.3.0",
-    "@checkstack/signal-common": "0.2.15",
-    "@checkstack/status-page-backend": "0.4.5",
-    "@checkstack/status-page-common": "0.5.1",
+    "@checkstack/ai-backend": "0.10.7",
+    "@checkstack/ai-common": "0.6.5",
+    "@checkstack/automation-backend": "0.10.9",
+    "@checkstack/backend-api": "0.29.1",
+    "@checkstack/cache-api": "0.3.18",
+    "@checkstack/cache-utils": "0.2.23",
+    "@checkstack/catalog-backend": "1.6.7",
+    "@checkstack/catalog-common": "2.6.2",
+    "@checkstack/command-backend": "0.2.19",
+    "@checkstack/common": "0.21.0",
+    "@checkstack/gitops-backend": "0.5.19",
+    "@checkstack/gitops-common": "0.7.2",
+    "@checkstack/healthcheck-common": "1.13.0",
+    "@checkstack/incident-backend": "1.9.5",
+    "@checkstack/incident-common": "1.7.2",
+    "@checkstack/maintenance-common": "1.8.2",
+    "@checkstack/notification-common": "1.5.2",
+    "@checkstack/queue-api": "0.3.18",
+    "@checkstack/satellite-backend": "0.8.1",
+    "@checkstack/script-packages-backend": "0.3.23",
+    "@checkstack/sdk": "0.123.1",
+    "@checkstack/secrets-backend": "0.3.1",
+    "@checkstack/secrets-common": "0.3.1",
+    "@checkstack/signal-common": "0.2.16",
+    "@checkstack/status-page-backend": "0.4.6",
+    "@checkstack/status-page-common": "0.5.2",
     "@hono/zod-validator": "^0.7.6",
     "@orpc/contract": "^1.14.4",
     "@orpc/server": "^1.14.4",
@@ -52,8 +52,8 @@
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
-    "@checkstack/scripts": "0.7.1",
-    "@checkstack/test-utils-backend": "0.1.52",
+    "@checkstack/scripts": "0.7.2",
+    "@checkstack/test-utils-backend": "0.1.53",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/aggregation-utils.test.ts

@@ -4,12 +4,14 @@ import {
   calculateLatencyStats,
   countStatuses,
   extractLatencies,
+  foldRunAssertionStats,
   mergeTieredBuckets,
   combineBuckets,
   reaggregateBuckets,
   mergeAggregatedBucketResults,
   type NormalizedBucket,
 } from "./aggregation-utils";
+import { computeAssertionKey } from "@checkstack/healthcheck-common";
 import {
   VersionedAggregated,
   aggregatedCounter,
@@ -982,3 +984,133 @@ describe("aggregation-utils", () => {
     });
   });
 });
+
+describe("assertion stats in aggregation", () => {
+  const KEY = computeAssertionKey({
+    assertion: { field: "statusCode", operator: "equals", value: 200 },
+  });
+
+  const outcome = (passed: boolean) => ({
+    key: KEY,
+    field: "statusCode",
+    operator: "equals",
+    value: "200",
+    passed,
+  });
+
+  describe("foldRunAssertionStats", () => {
+    it("folds outcomes across a bucket's runs per collector entry", () => {
+      const runs = [
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": { _collectorId: "c", _assertions: [outcome(true)] },
+            },
+          },
+        },
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": { _collectorId: "c", _assertions: [outcome(false)] },
+            },
+          },
+        },
+        // Pre-feature run without outcomes is tolerated.
+        { metadata: { collectors: { "uuid-1": { _collectorId: "c" } } } },
+      ];
+      expect(foldRunAssertionStats(runs)).toEqual({
+        "uuid-1": { [KEY]: { passCount: 1, failCount: 1 } },
+      });
+    });
+
+    it("returns undefined when no run carries outcomes", () => {
+      expect(foldRunAssertionStats([{ metadata: {} }, {}])).toBeUndefined();
+    });
+
+    it("ignores malformed outcome entries", () => {
+      const runs = [
+        {
+          metadata: {
+            collectors: {
+              "uuid-1": {
+                _collectorId: "c",
+                _assertions: ["garbage", outcome(true)],
+              },
+            },
+          },
+        },
+      ];
+      expect(foldRunAssertionStats(runs)).toEqual({
+        "uuid-1": { [KEY]: { passCount: 1, failCount: 0 } },
+      });
+    });
+  });
+
+  describe("mergeAggregatedBucketResults with assertion stats", () => {
+    it("merges counts additively and keeps them out of strategy merging", () => {
+      const mergeAggregatedStates = mock(
+        (a: Record<string, unknown>, b: Record<string, unknown>) => ({
+          ...a,
+          ...b,
+        }),
+      );
+      const registry = {
+        getStrategy: mock(() => ({
+          aggregatedResult: { mergeAggregatedStates },
+        })),
+        register: mock(() => {}),
+        getStrategies: mock(() => []),
+        getStrategiesWithMeta: mock(() => []),
+      } as unknown as HealthCheckRegistry;
+      const collectorRegistry = {
+        register: mock(() => {}),
+        getCollector: mock(() => undefined),
+        getCollectors: mock(() => []),
+      } as unknown as CollectorRegistry;
+
+      const merged = mergeAggregatedBucketResults({
+        aggregatedResults: [
+          {
+            uptime: 1,
+            assertions: { "uuid-1": { [KEY]: { passCount: 3, failCount: 1 } } },
+          },
+          {
+            uptime: 2,
+            assertions: { "uuid-1": { [KEY]: { passCount: 2, failCount: 0 } } },
+          },
+        ],
+        collectorRegistry,
+        registry,
+        strategyId: "test-strategy",
+      });
+
+      expect(merged?.assertions).toEqual({
+        "uuid-1": { [KEY]: { passCount: 5, failCount: 1 } },
+      });
+      // The strategy merger only ever saw its own fields.
+      for (const call of mergeAggregatedStates.mock.calls) {
+        for (const arg of call) {
+          expect(
+            (arg as Record<string, unknown>).assertions,
+          ).toBeUndefined();
+        }
+      }
+    });
+
+    it("single-bucket results pass through with their stats intact", () => {
+      const { collectorRegistry, registry, strategyId } =
+        createMockRegistries();
+      const only = {
+        assertions: { "uuid-1": { [KEY]: { passCount: 7, failCount: 0 } } },
+      };
+      expect(
+        mergeAggregatedBucketResults({
+          aggregatedResults: [only],
+          collectorRegistry,
+          registry,
+          strategyId,
+        }),
+      ).toBe(only);
+    });
+  });
+});

package/src/aggregation-utils.ts

@@ -2,6 +2,15 @@ import type {
   CollectorRegistry,
   HealthCheckRegistry,
 } from "@checkstack/backend-api";
+import {
+  ASSERTIONS_AGG_KEY,
+  AssertionOutcomeSchema,
+  foldOutcomesIntoStats,
+  mergeAssertionStats,
+  readAssertionStats,
+  type AssertionOutcome,
+  type BucketAssertionStats,
+} from "@checkstack/healthcheck-common";
 
 // ===== Percentile Calculation =====
 
@@ -138,7 +147,8 @@ export function aggregateCollectorData(
       const existing = aggregatedByUuid.get(uuid)?.aggregated;
 
       // Strip internal fields from collector data
-      const { _collectorId, _assertionFailed, ...collectorMetadata } = data;
+      const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
+        data;
 
       // Call mergeResult to incrementally aggregate
       const merged = registered.collector.mergeResult(existing, {
@@ -166,6 +176,39 @@ export function aggregateCollectorData(
   return result;
 }
 
+/**
+ * Fold the structured assertion outcomes of a bucket's raw runs into the
+ * per-assertion pass/fail stats (the on-read raw tier's counterpart of the
+ * realtime hourly fold). Returns undefined when no run carries outcomes.
+ */
+export function foldRunAssertionStats(
+  runs: Array<{ metadata?: Record<string, unknown> }>,
+): BucketAssertionStats | undefined {
+  let stats: BucketAssertionStats | undefined;
+  for (const run of runs) {
+    const collectors = run.metadata?.collectors as
+      | Record<string, Record<string, unknown>>
+      | undefined;
+    if (!collectors) continue;
+    for (const [uuid, data] of Object.entries(collectors)) {
+      const rawOutcomes = data._assertions;
+      if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
+      const outcomes: AssertionOutcome[] = [];
+      for (const raw of rawOutcomes) {
+        const parsed = AssertionOutcomeSchema.safeParse(raw);
+        if (parsed.success) outcomes.push(parsed.data);
+      }
+      if (outcomes.length === 0) continue;
+      stats = foldOutcomesIntoStats({
+        stats,
+        collectorEntryId: uuid,
+        outcomes,
+      });
+    }
+  }
+  return stats;
+}
+
 // ===== Bucket Result Merging =====
 
 /**
@@ -201,15 +244,29 @@ export function mergeAggregatedBucketResults(params: {
     return validResults[0];
   }
 
+  // === Platform-level assertion stats (additive, never strategy-merged) ===
+  let mergedAssertionStats: BucketAssertionStats | undefined;
+  for (const result of validResults) {
+    mergedAssertionStats = mergeAssertionStats({
+      a: mergedAssertionStats,
+      b: readAssertionStats({ aggregatedResult: result }),
+    });
+  }
+
   // === Strategy-level field merging ===
   let mergedStrategyFields: Record<string, unknown> = {};
 
   const registeredStrategy = registry.getStrategy(strategyId);
   if (registeredStrategy?.aggregatedResult) {
-    // Extract strategy-level fields (everything except 'collectors')
+    // Extract strategy-level fields (everything except the platform-owned
+    // 'collectors' and 'assertions' keys)
     const strategyDataSets: Array<Record<string, unknown>> = [];
     for (const result of validResults) {
-      const { collectors: _collectors, ...strategyFields } = result;
+      const {
+        collectors: _collectors,
+        [ASSERTIONS_AGG_KEY]: _assertions,
+        ...strategyFields
+      } = result;
       if (Object.keys(strategyFields).length > 0) {
         strategyDataSets.push(strategyFields);
       }
@@ -233,7 +290,11 @@ export function mergeAggregatedBucketResults(params: {
     }
   } else {
     // Strategy not found - preserve strategy fields from first result
-    const { collectors: _collectors, ...firstStrategyFields } = validResults[0];
+    const {
+      collectors: _collectors,
+      [ASSERTIONS_AGG_KEY]: _assertions,
+      ...firstStrategyFields
+    } = validResults[0];
     mergedStrategyFields = firstStrategyFields;
   }
 
@@ -302,17 +363,20 @@ export function mergeAggregatedBucketResults(params: {
     }
   }
 
-  // Combine strategy fields and collector fields
+  // Combine strategy fields, collector fields, and assertion stats
   const hasCollectors = Object.keys(mergedCollectors).length > 0;
   const hasStrategyFields = Object.keys(mergedStrategyFields).length > 0;
 
-  if (!hasCollectors && !hasStrategyFields) {
+  if (!hasCollectors && !hasStrategyFields && mergedAssertionStats === undefined) {
     return undefined;
   }
 
   return {
     ...mergedStrategyFields,
     ...(hasCollectors ? { collectors: mergedCollectors } : {}),
+    ...(mergedAssertionStats === undefined
+      ? {}
+      : { [ASSERTIONS_AGG_KEY]: mergedAssertionStats }),
   };
 }
 

package/src/collector-assertions.test.ts

@@ -1,6 +1,13 @@
 import { describe, expect, it } from "bun:test";
 import type { CollectorAssertion } from "@checkstack/healthcheck-common";
-import { evaluateCollectorAssertions } from "./collector-assertions";
+import {
+  ASSERTION_ACTUAL_MAX_LENGTH,
+  computeAssertionKey,
+} from "@checkstack/healthcheck-common";
+import {
+  evaluateCollectorAssertionOutcomes,
+  evaluateCollectorAssertions,
+} from "./collector-assertions";
 
 /** The HTTP Request collector's result shape, as the executor sees it. */
 const httpResult = (body: string): Record<string, unknown> => ({
@@ -274,3 +281,92 @@ describe("evaluateCollectorAssertions", () => {
     });
   });
 });
+
+describe("evaluateCollectorAssertionOutcomes", () => {
+  it("evaluates ALL assertions instead of short-circuiting on failure", () => {
+    const result = httpResult('{"status":"degraded"}');
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: [
+          { field: "statusCode", operator: "equals", value: 500 }, // fails
+          { field: "success", operator: "isTrue" }, // passes
+          jsonPathAssertion("$.status", "equals", "ok"), // fails
+        ],
+        result,
+      });
+    expect(outcomes.length).toBe(3);
+    expect(outcomes.map((o) => o.passed)).toEqual([false, true, false]);
+    // The legacy string still reports the FIRST failure only.
+    expect(firstFailureMessage).toBe("statusCode equals 500");
+  });
+
+  it("captures observed values for plain and JSONPath assertions", () => {
+    const result = httpResult('{"status":"ok","items":[1,2]}');
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [
+        { field: "statusCode", operator: "equals", value: 200 },
+        jsonPathAssertion("$.status", "equals", "ok"),
+        jsonPathAssertion("$.items", "lengthEquals", 2),
+      ],
+      result,
+    });
+    expect(outcomes[0].actual).toBe("200");
+    expect(outcomes[1].actual).toBe("ok");
+    expect(outcomes[2].actual).toBe("[1,2]");
+    expect(outcomes.every((o) => o.passed)).toBe(true);
+  });
+
+  it("truncates long observed values", () => {
+    const longBody = JSON.stringify({ blob: "x".repeat(5000) });
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [
+        { field: "body", operator: "contains", value: "x" },
+      ],
+      result: httpResult(longBody),
+    });
+    expect(outcomes[0].passed).toBe(true);
+    expect(outcomes[0].actual?.length).toBe(ASSERTION_ACTUAL_MAX_LENGTH);
+  });
+
+  it("fail-closed diagnostics become failed outcomes with messages", () => {
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: [
+          { field: "body.$", jsonPath: "  ", operator: "exists" },
+          jsonPathAssertion("$.status", "equals", "ok"),
+        ],
+        result: httpResult("not json"),
+      });
+    expect(outcomes[0].passed).toBe(false);
+    expect(outcomes[0].message).toBe("missing JSONPath expression");
+    expect(outcomes[1].passed).toBe(false);
+    expect(outcomes[1].message).toBe('field "body" is not valid JSON');
+    expect(firstFailureMessage).toBe(
+      "body.$ exists (missing JSONPath expression)",
+    );
+  });
+
+  it("outcomes carry the canonical assertion identity key", () => {
+    const assertion: CollectorAssertion = {
+      field: "statusCode",
+      operator: "equals",
+      value: 200,
+    };
+    const { outcomes } = evaluateCollectorAssertionOutcomes({
+      assertions: [assertion],
+      result: httpResult("{}"),
+    });
+    expect(outcomes[0].key).toBe(computeAssertionKey({ assertion }));
+    expect(outcomes[0].value).toBe("200");
+  });
+
+  it("no assertions yields an empty evaluation", () => {
+    const { outcomes, firstFailureMessage } =
+      evaluateCollectorAssertionOutcomes({
+        assertions: undefined,
+        result: httpResult("{}"),
+      });
+    expect(outcomes).toEqual([]);
+    expect(firstFailureMessage).toBeUndefined();
+  });
+});