@checkstack/healthcheck-backend 1.15.0 → 1.16.0

package/src/retention-job.ts

@@ -8,6 +8,12 @@ import {
   DEFAULT_RETENTION_CONFIG,
 } from "./schema";
 import { eq, and, lt, sql, desc } from "drizzle-orm";
+import {
+  ASSERTIONS_AGG_KEY,
+  mergeAssertionStats,
+  readAssertionStats,
+  type BucketAssertionStats,
+} from "@checkstack/healthcheck-common";
 import type { QueueManager } from "@checkstack/queue-api";
 
 type Db = SafeDatabase<typeof schema>;
@@ -244,6 +250,8 @@ export interface HourlyAggregateRow {
   minLatencyMs: number | null;
   maxLatencyMs: number | null;
   p95LatencyMs: number | null;
+  /** Carried for the per-assertion pass/fail counts (additive across hours). */
+  aggregatedResult?: Record<string, unknown> | null;
 }
 
 /** A computed daily aggregate ready to upsert. */
@@ -261,6 +269,12 @@ export interface DailyAggregateValues {
   minLatencyMs: number | undefined;
   maxLatencyMs: number | undefined;
   p95LatencyMs: number | undefined;
+  /**
+   * Summed per-assertion pass/fail counts across the day's hourly buckets.
+   * The ONLY part of `aggregatedResult` that survives the daily rollup —
+   * assertion counts are purely additive, unlike strategy/collector states.
+   */
+  assertionStats: BucketAssertionStats | undefined;
 }
 
 /**
@@ -296,6 +310,7 @@ export function buildDailyAggregates(
     let degradedCount = 0;
     let unhealthyCount = 0;
     let latencySumMs = 0;
+    let assertionStats: BucketAssertionStats | undefined;
 
     for (const a of rows) {
       runCount += a.runCount;
@@ -308,6 +323,12 @@ export function buildDailyAggregates(
       } else if (a.avgLatencyMs !== null) {
         latencySumMs += a.avgLatencyMs * a.runCount;
       }
+      assertionStats = mergeAssertionStats({
+        a: assertionStats,
+        b: readAssertionStats({
+          aggregatedResult: a.aggregatedResult ?? undefined,
+        }),
+      });
     }
 
     const minValues = rows
@@ -336,6 +357,7 @@ export function buildDailyAggregates(
       maxLatencyMs: maxValues.length > 0 ? Math.max(...maxValues) : undefined,
       // Use max of hourly p95s as an upper-bound approximation.
       p95LatencyMs: p95Values.length > 0 ? Math.max(...p95Values) : undefined,
+      assertionStats,
     });
   }
 
@@ -370,6 +392,42 @@ async function rollupHourlyAggregates(params: RollupParams) {
   // Fold into daily aggregates, preserving (day, environmentId, sourceId) series.
   for (const daily of buildDailyAggregates(oldHourly)) {
     const newLatencySum = daily.latencySumMs;
+
+    // Assertion pass/fail counts are the ONLY aggregatedResult content that
+    // survives the daily rollup (strategy/collector states cannot combine
+    // across hours). The conflict path merges in JS by pre-reading the
+    // existing daily row — safe because retention runs in a single work-queue
+    // consumer group, so there is no concurrent writer for this tuple.
+    let dailyAggregatedResult: Record<string, unknown> | undefined;
+    if (daily.assertionStats !== undefined) {
+      const [existingDaily] = await db
+        .select({ aggregatedResult: healthCheckAggregates.aggregatedResult })
+        .from(healthCheckAggregates)
+        .where(
+          and(
+            eq(healthCheckAggregates.systemId, systemId),
+            eq(healthCheckAggregates.configurationId, configurationId),
+            eq(healthCheckAggregates.bucketSize, "daily"),
+            eq(healthCheckAggregates.bucketStart, daily.bucketStart),
+            daily.environmentId === null
+              ? sql`${healthCheckAggregates.environmentId} IS NULL`
+              : eq(healthCheckAggregates.environmentId, daily.environmentId),
+            daily.sourceId === null
+              ? sql`${healthCheckAggregates.sourceId} IS NULL`
+              : eq(healthCheckAggregates.sourceId, daily.sourceId),
+          ),
+        );
+      const mergedStats = mergeAssertionStats({
+        a: readAssertionStats({
+          aggregatedResult: existingDaily?.aggregatedResult ?? undefined,
+        }),
+        b: daily.assertionStats,
+      });
+      if (mergedStats !== undefined) {
+        dailyAggregatedResult = { [ASSERTIONS_AGG_KEY]: mergedStats };
+      }
+    }
+
     // Upsert the daily aggregate. A row may already exist for this
     // (configurationId, systemId, environmentId, day, daily, sourceId) tuple if
     // a prior rollup ran and then late-arriving hourly buckets (e.g. from a
@@ -394,7 +452,8 @@ async function rollupHourlyAggregates(params: RollupParams) {
         minLatencyMs: daily.minLatencyMs,
         maxLatencyMs: daily.maxLatencyMs,
         p95LatencyMs: daily.p95LatencyMs,
-        aggregatedResult: undefined, // Cannot combine result across hours
+        // Only the additive assertion counts survive across hours.
+        aggregatedResult: dailyAggregatedResult,
       })
       .onConflictDoUpdate({
         target: [...DAILY_AGGREGATE_CONFLICT_TARGET],
@@ -417,6 +476,10 @@ async function rollupHourlyAggregates(params: RollupParams) {
             daily.p95LatencyMs === undefined
               ? sql`${healthCheckAggregates.p95LatencyMs}`
               : sql`GREATEST(COALESCE(${healthCheckAggregates.p95LatencyMs}, ${daily.p95LatencyMs}), ${daily.p95LatencyMs})`,
+          // JS-merged above (existing row's counts + this rollup's counts).
+          ...(dailyAggregatedResult === undefined
+            ? {}
+            : { aggregatedResult: dailyAggregatedResult }),
         },
       });
   }

package/src/retention-rollup.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect } from "bun:test";
+import { computeAssertionKey } from "@checkstack/healthcheck-common";
 import { getTableConfig } from "drizzle-orm/pg-core";
 import { healthCheckAggregates } from "./schema";
 import {
@@ -116,3 +117,71 @@ describe("DAILY_AGGREGATE_CONFLICT_TARGET", () => {
     expect(targetCols).toEqual(constraintCols);
   });
 });
+
+describe("buildDailyAggregates - assertion stats", () => {
+  const KEY = computeAssertionKey({
+    assertion: { field: "statusCode", operator: "equals", value: 200 },
+  });
+
+  it("sums per-assertion counts across the day's hourly buckets", () => {
+    const daily = buildDailyAggregates([
+      hourly({
+        bucketStart: new Date("2026-01-01T03:00:00.000Z"),
+        aggregatedResult: {
+          collectors: {},
+          assertions: { "uuid-1": { [KEY]: { passCount: 50, failCount: 2 } } },
+        },
+      }),
+      hourly({
+        bucketStart: new Date("2026-01-01T04:00:00.000Z"),
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 60, failCount: 0 } } },
+        },
+      }),
+      // Pre-feature hourly bucket without stats is tolerated.
+      hourly({
+        bucketStart: new Date("2026-01-01T05:00:00.000Z"),
+        aggregatedResult: null,
+      }),
+    ]);
+
+    expect(daily.length).toBe(1);
+    expect(daily[0].assertionStats).toEqual({
+      "uuid-1": { [KEY]: { passCount: 110, failCount: 2 } },
+    });
+  });
+
+  it("keeps assertion stats scoped to their (env, source) series", () => {
+    const daily = buildDailyAggregates([
+      hourly({
+        environmentId: "prod",
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 1, failCount: 0 } } },
+        },
+      }),
+      hourly({
+        environmentId: "staging",
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 0, failCount: 1 } } },
+        },
+      }),
+    ]);
+
+    expect(daily.length).toBe(2);
+    const prod = daily.find((d) => d.environmentId === "prod");
+    const staging = daily.find((d) => d.environmentId === "staging");
+    expect(prod?.assertionStats?.["uuid-1"][KEY]).toEqual({
+      passCount: 1,
+      failCount: 0,
+    });
+    expect(staging?.assertionStats?.["uuid-1"][KEY]).toEqual({
+      passCount: 0,
+      failCount: 1,
+    });
+  });
+
+  it("buckets without stats yield undefined assertionStats", () => {
+    const daily = buildDailyAggregates([hourly({})]);
+    expect(daily[0].assertionStats).toBeUndefined();
+  });
+});

package/src/service-ingest-assertions.test.ts

@@ -0,0 +1,213 @@
+import { describe, expect, it, mock } from "bun:test";
+import { z } from "zod";
+import { Versioned } from "@checkstack/backend-api";
+import {
+  computeAssertionKey,
+  healthResultNumber,
+  healthResultString,
+  type CollectorConfigEntry,
+} from "@checkstack/healthcheck-common";
+import { HealthCheckService } from "./service";
+
+/**
+ * Satellite ingest evaluates assertions ON THE CORE (satellites never held
+ * the assertion semantics — before this, satellite-executed checks silently
+ * skipped assertions), then strips ephemeral fields for parity with local
+ * runs. These tests drive `ingestSatelliteResult` against a mock db and
+ * assert on the run row it persists.
+ */
+
+const KEY = computeAssertionKey({
+  assertion: { field: "statusCode", operator: "equals", value: 200 },
+});
+
+// `body` is ephemeral: assertable at evaluation time, never persisted.
+const collectorResultSchema = z.object({
+  statusCode: healthResultNumber({
+    "x-chart-type": "counter",
+    "x-anomaly-enabled": false,
+  }),
+  body: healthResultString({ "x-ephemeral": true }),
+});
+
+function buildService({
+  entries,
+  inserted,
+}: {
+  entries: CollectorConfigEntry[];
+  inserted: Record<string, unknown>[];
+}) {
+  const tx = {
+    insert: mock(() => ({
+      values: mock((vals: Record<string, unknown>) => {
+        inserted.push(vals);
+        return Object.assign(Promise.resolve(), {
+          onConflictDoUpdate: mock(() => Promise.resolve()),
+          onConflictDoNothing: mock(() => Promise.resolve()),
+        });
+      }),
+    })),
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() =>
+          Object.assign(Promise.resolve([]), {
+            limit: mock(() => Promise.resolve([])),
+          }),
+        ),
+      })),
+    })),
+  };
+
+  const db = {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() => Promise.resolve([{ collectors: entries }])),
+      })),
+    })),
+    transaction: mock(async (fn: (t: typeof tx) => Promise<void>) => fn(tx)),
+  };
+
+  const collectorRegistry = {
+    register: mock(() => {}),
+    getCollector: mock(() => ({
+      collector: {
+        id: "test-collector",
+        result: new Versioned({ version: 1, schema: collectorResultSchema }),
+      },
+    })),
+    getCollectors: mock(() => []),
+  };
+
+  return new HealthCheckService(
+    db as unknown as ConstructorParameters<typeof HealthCheckService>[0],
+    {} as unknown as ConstructorParameters<typeof HealthCheckService>[1],
+    collectorRegistry as unknown as ConstructorParameters<
+      typeof HealthCheckService
+    >[2],
+  );
+}
+
+function satelliteResult({ statusCode }: { statusCode: number }) {
+  return {
+    status: "healthy",
+    latencyMs: 42,
+    message: "Completed in 42ms",
+    metadata: {
+      collectors: {
+        "entry-1": {
+          _collectorId: "test-collector",
+          statusCode,
+          body: '{"status":"ok"}',
+        },
+      },
+    },
+  };
+}
+
+const entries: CollectorConfigEntry[] = [
+  {
+    id: "entry-1",
+    collectorId: "test-collector",
+    config: {},
+    assertions: [{ field: "statusCode", operator: "equals", value: 200 }],
+  },
+];
+
+async function ingest({
+  entries,
+  statusCode,
+}: {
+  entries: CollectorConfigEntry[];
+  statusCode: number;
+}) {
+  const inserted: Record<string, unknown>[] = [];
+  const service = buildService({ entries, inserted });
+  await service.ingestSatelliteResult({
+    configId: "config-1",
+    systemId: "system-1",
+    status: "healthy",
+    latencyMs: 42,
+    result: satelliteResult({ statusCode }) as never,
+    executedAt: "2026-07-03T10:00:00.000Z",
+    sourceId: "sat-1",
+    sourceLabel: "EU West",
+  });
+  const runInsert = inserted.find((v) => "status" in v && "result" in v);
+  expect(runInsert).toBeDefined();
+  return runInsert as Record<string, unknown>;
+}
+
+function collectorEntryOf(runInsert: Record<string, unknown>) {
+  const result = runInsert.result as {
+    metadata: { collectors: Record<string, Record<string, unknown>> };
+  };
+  return result.metadata.collectors["entry-1"];
+}
+
+describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
+  it("downgrades a satellite-healthy run whose assertion fails", async () => {
+    const runInsert = await ingest({ entries, statusCode: 404 });
+    expect(runInsert.status).toBe("unhealthy");
+
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertionFailed).toBe("statusCode equals 200");
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ key: KEY, passed: false, actual: "404" }),
+    ]);
+    const message = (runInsert.result as { message: string }).message;
+    expect(message).toBe(
+      "Check failed: Assertion failed: statusCode equals 200",
+    );
+  });
+
+  it("keeps a passing run healthy and stores the passing outcome", async () => {
+    const runInsert = await ingest({ entries, statusCode: 200 });
+    expect(runInsert.status).toBe("healthy");
+
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertionFailed).toBeUndefined();
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ key: KEY, passed: true, actual: "200" }),
+    ]);
+  });
+
+  it("strips ephemeral fields AFTER assertions ran against them", async () => {
+    const withBodyAssertion: CollectorConfigEntry[] = [
+      {
+        id: "entry-1",
+        collectorId: "test-collector",
+        config: {},
+        assertions: [
+          {
+            field: "body.$",
+            jsonPath: "$.status",
+            operator: "equals",
+            value: "ok",
+          },
+        ],
+      },
+    ];
+    const runInsert = await ingest({
+      entries: withBodyAssertion,
+      statusCode: 200,
+    });
+
+    const entry = collectorEntryOf(runInsert);
+    // The JSONPath assertion evaluated against the (ephemeral) body...
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ passed: true, actual: "ok" }),
+    ]);
+    // ...but the body itself never reaches storage.
+    expect(entry.body).toBeUndefined();
+    expect(entry.statusCode).toBe(200);
+    expect(runInsert.status).toBe("healthy");
+  });
+
+  it("tolerates collector entries the config no longer knows", async () => {
+    const runInsert = await ingest({ entries: [], statusCode: 500 });
+    // No assertions configured: status passes through untouched.
+    expect(runInsert.status).toBe("healthy");
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertions).toBeUndefined();
+  });
+});

package/src/service.ts

@@ -12,7 +12,9 @@ import {
   type CollectorConfigEntry,
   type HealthcheckSignalStatuses,
   type RunStats,
+  stripEphemeralFields,
 } from "@checkstack/healthcheck-common";
+import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
 import { summarizeRuns, type StatRun } from "./run-stats.logic";
 import type { ConfigService } from "@checkstack/backend-api";
 import type { InferClient } from "@checkstack/common";
@@ -57,12 +59,14 @@ import type {
 import {
   aggregateCollectorData,
   extractLatencies,
+  foldRunAssertionStats,
   mergeTieredBuckets,
   reaggregateBuckets,
   countStatuses,
   calculateLatencyStats,
   type NormalizedBucket,
 } from "./aggregation-utils";
+import { ASSERTIONS_AGG_KEY } from "@checkstack/healthcheck-common";
 import {
   extractConfigurationSecrets,
   mergeConfigurationSecrets,
@@ -2151,9 +2155,16 @@ export class HealthCheckService {
           );
         }
 
+        // Per-assertion pass/fail counts (platform-owned, sibling of
+        // `collectors` — see assertion-analytics in healthcheck-common).
+        const assertionStats = foldRunAssertionStats(bucket.runs);
+
         aggregatedResult = {
           ...strategyResult,
           ...(collectorsAggregated ? { collectors: collectorsAggregated } : {}),
+          ...(assertionStats === undefined
+            ? {}
+            : { [ASSERTIONS_AGG_KEY]: assertionStats }),
         };
       }
 
@@ -2421,6 +2432,15 @@ export class HealthCheckService {
    * Ingest a health check result from a satellite.
    * Stores the run with source attribution (sourceId + sourceLabel)
    * and triggers incremental aggregation to keep charts/availability current.
+   *
+   * Assertions are evaluated HERE, on the core, not on the satellite: the
+   * satellite never held the assertion semantics, so historically
+   * satellite-executed checks silently skipped assertions entirely.
+   * Evaluating at ingest fixes that for every satellite version with no
+   * wire-protocol change. Caveat: buffered results are evaluated against the
+   * configuration CURRENT at ingest time. Ephemeral result fields (e.g. raw
+   * HTTP bodies) are needed for JSONPath assertions and are stripped right
+   * after evaluation, matching what the local executor stores.
    */
   async ingestSatelliteResult(props: {
     configId: string;
@@ -2432,20 +2452,77 @@ export class HealthCheckService {
     sourceId: string;
     sourceLabel: string;
   }) {
-    const {
-      configId,
-      systemId,
-      status,
-      latencyMs,
-      result,
-      sourceId,
-      sourceLabel,
-    } = props;
+    const { configId, systemId, latencyMs, result, sourceId, sourceLabel } =
+      props;
 
     const resultRecord = result
       ? ({ ...result } as Record<string, unknown>)
       : {};
 
+    let status = props.status;
+    const metadata = resultRecord.metadata as
+      | Record<string, unknown>
+      | undefined;
+    const collectorsMeta = metadata?.collectors as
+      | Record<string, Record<string, unknown>>
+      | undefined;
+    if (collectorsMeta && Object.keys(collectorsMeta).length > 0) {
+      const [configRow] = await this.db
+        .select({ collectors: healthCheckConfigurations.collectors })
+        .from(healthCheckConfigurations)
+        .where(eq(healthCheckConfigurations.id, configId));
+      const entries: CollectorConfigEntry[] = configRow?.collectors ?? [];
+
+      let firstFailure: string | undefined;
+      const nextCollectorsMeta: Record<string, Record<string, unknown>> = {
+        ...collectorsMeta,
+      };
+      for (const entry of entries) {
+        const entryResult = nextCollectorsMeta[entry.id];
+        if (!entryResult || typeof entryResult !== "object") continue;
+
+        let evaluated: Record<string, unknown> = { ...entryResult };
+        if (entry.assertions?.length) {
+          const evaluation = evaluateCollectorAssertionOutcomes({
+            assertions: entry.assertions,
+            result: evaluated,
+          });
+          evaluated._assertions = evaluation.outcomes;
+          evaluated._assertionFailed = evaluation.firstFailureMessage;
+          if (
+            evaluation.firstFailureMessage !== undefined &&
+            firstFailure === undefined
+          ) {
+            firstFailure = evaluation.firstFailureMessage;
+          }
+        }
+
+        // Parity with the local executor: satellites send raw results, so
+        // ephemeral fields (assertable but never persisted) get stripped
+        // here, AFTER assertions ran against them.
+        const registered = this.collectorRegistry.getCollector(
+          entry.collectorId,
+        );
+        if (registered) {
+          evaluated = stripEphemeralFields(
+            evaluated,
+            registered.collector.result.schema,
+          );
+        }
+        nextCollectorsMeta[entry.id] = evaluated;
+      }
+
+      resultRecord.metadata = { ...metadata, collectors: nextCollectorsMeta };
+
+      // Mirror the local executor: a failed assertion downgrades a run the
+      // satellite reported healthy.
+      if (firstFailure !== undefined && status === "healthy") {
+        status = "unhealthy";
+        resultRecord.status = status;
+        resultRecord.message = `Check failed: Assertion failed: ${firstFailure}`;
+      }
+    }
+
     // Atomic: the run row and the hourly-aggregate increment it feeds must
     // commit together. Without the transaction a failure on the (non-idempotent
     // `runCount + 1`) aggregate left a committed run that the aggregate never