@checkstack/healthcheck-backend 1.15.0 → 1.17.0

package/src/realtime-aggregation.ts

@@ -1,4 +1,11 @@
 import type { SafeDatabase, CollectorRegistry } from "@checkstack/backend-api";
+import {
+  ASSERTIONS_AGG_KEY,
+  AssertionOutcomeSchema,
+  foldOutcomesIntoStats,
+  readAssertionStats,
+  type AssertionOutcome,
+} from "@checkstack/healthcheck-common";
 import { TDigest } from "tdigest";
 import * as schema from "./schema";
 import { healthCheckAggregates } from "./schema";
@@ -269,6 +276,31 @@ function mergeCollectorResults(params: {
     return existingResult ?? undefined;
   }
 
+  // Fold this run's structured assertion outcomes into the bucket's
+  // per-assertion pass/fail counts (a PLATFORM-owned top-level key, sibling
+  // of `collectors`, so collector mergers never see it). Folded for every
+  // entry that carries outcomes, even collectors without a mergeResult.
+  let assertionStats = readAssertionStats({
+    aggregatedResult: existingResult ?? undefined,
+  });
+  if (runCollectors) {
+    for (const [uuid, collectorData] of Object.entries(runCollectors)) {
+      const rawOutcomes = collectorData._assertions;
+      if (!Array.isArray(rawOutcomes) || rawOutcomes.length === 0) continue;
+      const outcomes: AssertionOutcome[] = [];
+      for (const raw of rawOutcomes) {
+        const parsed = AssertionOutcomeSchema.safeParse(raw);
+        if (parsed.success) outcomes.push(parsed.data);
+      }
+      if (outcomes.length === 0) continue;
+      assertionStats = foldOutcomesIntoStats({
+        stats: assertionStats,
+        collectorEntryId: uuid,
+        outcomes,
+      });
+    }
+  }
+
   // Start with existing collectors or empty object
   const existingCollectors = (existingResult?.collectors ?? {}) as Record<
     string,
@@ -291,7 +323,7 @@ function mergeCollectorResults(params: {
       const existingAggregate = existingCollectors[uuid];
 
       // Strip internal fields from collector data for the run
-      const { _collectorId, _assertionFailed, ...collectorMetadata } =
+      const { _collectorId, _assertionFailed, _assertions, ...collectorMetadata } =
         collectorData;
 
       // Call the collector's mergeResult
@@ -313,5 +345,10 @@ function mergeCollectorResults(params: {
     }
   }
 
-  return { collectors: mergedCollectors };
+  return {
+    collectors: mergedCollectors,
+    ...(assertionStats === undefined
+      ? {}
+      : { [ASSERTIONS_AGG_KEY]: assertionStats }),
+  };
 }

package/src/retention-job.ts

@@ -8,6 +8,12 @@ import {
   DEFAULT_RETENTION_CONFIG,
 } from "./schema";
 import { eq, and, lt, sql, desc } from "drizzle-orm";
+import {
+  ASSERTIONS_AGG_KEY,
+  mergeAssertionStats,
+  readAssertionStats,
+  type BucketAssertionStats,
+} from "@checkstack/healthcheck-common";
 import type { QueueManager } from "@checkstack/queue-api";
 
 type Db = SafeDatabase<typeof schema>;
@@ -244,6 +250,8 @@ export interface HourlyAggregateRow {
   minLatencyMs: number | null;
   maxLatencyMs: number | null;
   p95LatencyMs: number | null;
+  /** Carried for the per-assertion pass/fail counts (additive across hours). */
+  aggregatedResult?: Record<string, unknown> | null;
 }
 
 /** A computed daily aggregate ready to upsert. */
@@ -261,6 +269,12 @@ export interface DailyAggregateValues {
   minLatencyMs: number | undefined;
   maxLatencyMs: number | undefined;
   p95LatencyMs: number | undefined;
+  /**
+   * Summed per-assertion pass/fail counts across the day's hourly buckets.
+   * The ONLY part of `aggregatedResult` that survives the daily rollup —
+   * assertion counts are purely additive, unlike strategy/collector states.
+   */
+  assertionStats: BucketAssertionStats | undefined;
 }
 
 /**
@@ -296,6 +310,7 @@ export function buildDailyAggregates(
     let degradedCount = 0;
     let unhealthyCount = 0;
     let latencySumMs = 0;
+    let assertionStats: BucketAssertionStats | undefined;
 
     for (const a of rows) {
       runCount += a.runCount;
@@ -308,6 +323,12 @@ export function buildDailyAggregates(
       } else if (a.avgLatencyMs !== null) {
         latencySumMs += a.avgLatencyMs * a.runCount;
       }
+      assertionStats = mergeAssertionStats({
+        a: assertionStats,
+        b: readAssertionStats({
+          aggregatedResult: a.aggregatedResult ?? undefined,
+        }),
+      });
     }
 
     const minValues = rows
@@ -336,6 +357,7 @@ export function buildDailyAggregates(
       maxLatencyMs: maxValues.length > 0 ? Math.max(...maxValues) : undefined,
       // Use max of hourly p95s as an upper-bound approximation.
       p95LatencyMs: p95Values.length > 0 ? Math.max(...p95Values) : undefined,
+      assertionStats,
     });
   }
 
@@ -370,6 +392,42 @@ async function rollupHourlyAggregates(params: RollupParams) {
   // Fold into daily aggregates, preserving (day, environmentId, sourceId) series.
   for (const daily of buildDailyAggregates(oldHourly)) {
     const newLatencySum = daily.latencySumMs;
+
+    // Assertion pass/fail counts are the ONLY aggregatedResult content that
+    // survives the daily rollup (strategy/collector states cannot combine
+    // across hours). The conflict path merges in JS by pre-reading the
+    // existing daily row — safe because retention runs in a single work-queue
+    // consumer group, so there is no concurrent writer for this tuple.
+    let dailyAggregatedResult: Record<string, unknown> | undefined;
+    if (daily.assertionStats !== undefined) {
+      const [existingDaily] = await db
+        .select({ aggregatedResult: healthCheckAggregates.aggregatedResult })
+        .from(healthCheckAggregates)
+        .where(
+          and(
+            eq(healthCheckAggregates.systemId, systemId),
+            eq(healthCheckAggregates.configurationId, configurationId),
+            eq(healthCheckAggregates.bucketSize, "daily"),
+            eq(healthCheckAggregates.bucketStart, daily.bucketStart),
+            daily.environmentId === null
+              ? sql`${healthCheckAggregates.environmentId} IS NULL`
+              : eq(healthCheckAggregates.environmentId, daily.environmentId),
+            daily.sourceId === null
+              ? sql`${healthCheckAggregates.sourceId} IS NULL`
+              : eq(healthCheckAggregates.sourceId, daily.sourceId),
+          ),
+        );
+      const mergedStats = mergeAssertionStats({
+        a: readAssertionStats({
+          aggregatedResult: existingDaily?.aggregatedResult ?? undefined,
+        }),
+        b: daily.assertionStats,
+      });
+      if (mergedStats !== undefined) {
+        dailyAggregatedResult = { [ASSERTIONS_AGG_KEY]: mergedStats };
+      }
+    }
+
     // Upsert the daily aggregate. A row may already exist for this
     // (configurationId, systemId, environmentId, day, daily, sourceId) tuple if
     // a prior rollup ran and then late-arriving hourly buckets (e.g. from a
@@ -394,7 +452,8 @@ async function rollupHourlyAggregates(params: RollupParams) {
         minLatencyMs: daily.minLatencyMs,
         maxLatencyMs: daily.maxLatencyMs,
         p95LatencyMs: daily.p95LatencyMs,
-        aggregatedResult: undefined, // Cannot combine result across hours
+        // Only the additive assertion counts survive across hours.
+        aggregatedResult: dailyAggregatedResult,
       })
       .onConflictDoUpdate({
         target: [...DAILY_AGGREGATE_CONFLICT_TARGET],
@@ -417,6 +476,10 @@ async function rollupHourlyAggregates(params: RollupParams) {
             daily.p95LatencyMs === undefined
               ? sql`${healthCheckAggregates.p95LatencyMs}`
               : sql`GREATEST(COALESCE(${healthCheckAggregates.p95LatencyMs}, ${daily.p95LatencyMs}), ${daily.p95LatencyMs})`,
+          // JS-merged above (existing row's counts + this rollup's counts).
+          ...(dailyAggregatedResult === undefined
+            ? {}
+            : { aggregatedResult: dailyAggregatedResult }),
         },
       });
   }

package/src/retention-rollup.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect } from "bun:test";
+import { computeAssertionKey } from "@checkstack/healthcheck-common";
 import { getTableConfig } from "drizzle-orm/pg-core";
 import { healthCheckAggregates } from "./schema";
 import {
@@ -116,3 +117,71 @@ describe("DAILY_AGGREGATE_CONFLICT_TARGET", () => {
     expect(targetCols).toEqual(constraintCols);
   });
 });
+
+describe("buildDailyAggregates - assertion stats", () => {
+  const KEY = computeAssertionKey({
+    assertion: { field: "statusCode", operator: "equals", value: 200 },
+  });
+
+  it("sums per-assertion counts across the day's hourly buckets", () => {
+    const daily = buildDailyAggregates([
+      hourly({
+        bucketStart: new Date("2026-01-01T03:00:00.000Z"),
+        aggregatedResult: {
+          collectors: {},
+          assertions: { "uuid-1": { [KEY]: { passCount: 50, failCount: 2 } } },
+        },
+      }),
+      hourly({
+        bucketStart: new Date("2026-01-01T04:00:00.000Z"),
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 60, failCount: 0 } } },
+        },
+      }),
+      // Pre-feature hourly bucket without stats is tolerated.
+      hourly({
+        bucketStart: new Date("2026-01-01T05:00:00.000Z"),
+        aggregatedResult: null,
+      }),
+    ]);
+
+    expect(daily.length).toBe(1);
+    expect(daily[0].assertionStats).toEqual({
+      "uuid-1": { [KEY]: { passCount: 110, failCount: 2 } },
+    });
+  });
+
+  it("keeps assertion stats scoped to their (env, source) series", () => {
+    const daily = buildDailyAggregates([
+      hourly({
+        environmentId: "prod",
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 1, failCount: 0 } } },
+        },
+      }),
+      hourly({
+        environmentId: "staging",
+        aggregatedResult: {
+          assertions: { "uuid-1": { [KEY]: { passCount: 0, failCount: 1 } } },
+        },
+      }),
+    ]);
+
+    expect(daily.length).toBe(2);
+    const prod = daily.find((d) => d.environmentId === "prod");
+    const staging = daily.find((d) => d.environmentId === "staging");
+    expect(prod?.assertionStats?.["uuid-1"][KEY]).toEqual({
+      passCount: 1,
+      failCount: 0,
+    });
+    expect(staging?.assertionStats?.["uuid-1"][KEY]).toEqual({
+      passCount: 0,
+      failCount: 1,
+    });
+  });
+
+  it("buckets without stats yield undefined assertionStats", () => {
+    const daily = buildDailyAggregates([hourly({})]);
+    expect(daily[0].assertionStats).toBeUndefined();
+  });
+});

package/src/router.ts

@@ -39,6 +39,11 @@ import { CatalogApi } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import type { Logger } from "@checkstack/backend-api";
 import type { HealthCheckCache } from "./cache";
+import {
+  applySystemHealthOverrides,
+  type SystemHealthOverrideReader,
+} from "./system-health-override";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
 
 /**
  * Creates the healthcheck router using contract-based implementation.
@@ -81,6 +86,17 @@ export const createHealthCheckRouter = (opts: {
    * router MUST receive it or writes would store inline secrets verbatim.
    */
   configSecrets?: HealthCheckSecretsDeps;
+  /**
+   * Reads active incident health overrides and folds them into the two
+   * user-facing system-health reads (single + bulk) via worst-wins, so a system
+   * shows the status an active incident forces even when its checks look fine.
+   * Applied OUTSIDE the status cache (always live, so an override lifts the
+   * instant its incident resolves) and ONLY in these RPC handlers - never in the
+   * shared `getSystemHealthStatus` deriver, whose other callers (SLO downtime,
+   * the AI signals scan, the persisted `health` entity) must stay checks-only.
+   * Optional so tests / no-incident deployments simply skip the fold.
+   */
+  incidentHealthOverrideReader?: SystemHealthOverrideReader;
 }) => {
   const {
     database,
@@ -94,6 +110,7 @@ export const createHealthCheckRouter = (opts: {
     logger,
     signalService,
     recomputeSystemRollupHealth,
+    incidentHealthOverrideReader,
   } = opts;
   // Create service instance once - shared across all handlers
   const service = new HealthCheckService(
@@ -105,6 +122,43 @@ export const createHealthCheckRouter = (opts: {
     opts.configSecrets,
   );
 
+  /**
+   * Fold active incident health overrides into a batch of checks-only system
+   * statuses via worst-wins. Reads overrides for all systems in ONE incident RPC
+   * (or none, when no reader is wired). Resilient by design: incidents are a
+   * best-effort enrichment of health, so if the read fails the checks-only
+   * statuses are returned unchanged rather than failing the whole health read.
+   */
+  const foldIncidentOverrides = async (
+    statuses: Record<string, SystemHealthStatusResponse>,
+  ): Promise<Record<string, SystemHealthStatusResponse>> => {
+    const systemIds = Object.keys(statuses);
+    if (!incidentHealthOverrideReader || systemIds.length === 0) {
+      return statuses;
+    }
+    let overridesBySystem: Awaited<
+      ReturnType<SystemHealthOverrideReader["getActiveOverrides"]>
+    >;
+    try {
+      overridesBySystem =
+        await incidentHealthOverrideReader.getActiveOverrides(systemIds);
+    } catch (error) {
+      logger.warn(
+        "Failed to read incident health overrides; returning checks-only status",
+        { error: extractErrorMessage(error) },
+      );
+      return statuses;
+    }
+    const folded: Record<string, SystemHealthStatusResponse> = {};
+    for (const [systemId, base] of Object.entries(statuses)) {
+      folded[systemId] = applySystemHealthOverrides({
+        base,
+        overrides: overridesBySystem[systemId] ?? [],
+      });
+    }
+    return folded;
+  };
+
   // Create contract implementer with context type AND auto auth middleware
   const os = implement(healthCheckContract)
     .$context<RpcContext>()
@@ -201,6 +255,7 @@ export const createHealthCheckRouter = (opts: {
         displayName: r.strategy.displayName,
         description: r.strategy.description,
         category: (r.strategy.category ?? "other") as StrategyCategory,
+        setupInstructions: r.strategy.setupInstructions,
         configSchema: toJsonSchema(r.strategy.config.schema),
         resultSchema: r.strategy.result
           ? toJsonSchemaWithChartMeta(r.strategy.result.schema)
@@ -622,9 +677,13 @@ export const createHealthCheckRouter = (opts: {
     ),
     getSystemHealthStatus: os.getSystemHealthStatus.handler(
       async ({ input }) => {
-        return cache.wrapSystemHealthStatus(input.systemId, () =>
+        const base = await cache.wrapSystemHealthStatus(input.systemId, () =>
           service.getSystemHealthStatus(input.systemId),
         );
+        const folded = await foldIncidentOverrides({
+          [input.systemId]: base,
+        });
+        return folded[input.systemId]!;
       },
     ),
 
@@ -634,10 +693,7 @@ export const createHealthCheckRouter = (opts: {
         // and invalidated by id on mutations, so dashboards with overlapping
         // (but non-identical) system sets share cache entries. See
         // ./cache.ts for the key/TTL/invalidation contract.
-        const statuses: Record<
-          string,
-          Awaited<ReturnType<typeof service.getSystemHealthStatus>>
-        > = {};
+        const statuses: Record<string, SystemHealthStatusResponse> = {};
         await Promise.all(
           input.systemIds.map(async (systemId) => {
             statuses[systemId] = await cache.wrapSystemHealthStatus(
@@ -646,6 +702,37 @@ export const createHealthCheckRouter = (opts: {
             );
           }),
         );
+        return { statuses: await foldIncidentOverrides(statuses) };
+      },
+    ),
+
+    getBulkSystemHealthMatrix: os.getBulkSystemHealthMatrix.handler(
+      async ({ input }) => {
+        const matrix = await service.getBulkSystemHealthMatrix(input.systemIds);
+
+        // Fold active incident overrides into each system's OVERALL rollup, so
+        // an incident-forced status still propagates through dependencies (as
+        // it does via getBulkSystemHealthStatus). Per-environment slices track
+        // health-check status only - incidents force whole-system health, which
+        // any-environment (env=null) dependency cells read from this rollup.
+        const overallOnly: Record<string, SystemHealthStatusResponse> = {};
+        for (const [systemId, m] of Object.entries(matrix)) {
+          overallOnly[systemId] = {
+            status: m.status,
+            evaluatedAt: new Date(),
+            checkStatuses: m.checkStatuses,
+          };
+        }
+        const folded = await foldIncidentOverrides(overallOnly);
+
+        const statuses: Record<string, (typeof matrix)[string]> = {};
+        for (const [systemId, m] of Object.entries(matrix)) {
+          statuses[systemId] = {
+            status: folded[systemId]?.status ?? m.status,
+            checkStatuses: m.checkStatuses,
+            environments: m.environments,
+          };
+        }
         return { statuses };
       },
     ),

package/src/service-ingest-assertions.test.ts

@@ -0,0 +1,213 @@
+import { describe, expect, it, mock } from "bun:test";
+import { z } from "zod";
+import { Versioned } from "@checkstack/backend-api";
+import {
+  computeAssertionKey,
+  healthResultNumber,
+  healthResultString,
+  type CollectorConfigEntry,
+} from "@checkstack/healthcheck-common";
+import { HealthCheckService } from "./service";
+
+/**
+ * Satellite ingest evaluates assertions ON THE CORE (satellites never held
+ * the assertion semantics — before this, satellite-executed checks silently
+ * skipped assertions), then strips ephemeral fields for parity with local
+ * runs. These tests drive `ingestSatelliteResult` against a mock db and
+ * assert on the run row it persists.
+ */
+
+const KEY = computeAssertionKey({
+  assertion: { field: "statusCode", operator: "equals", value: 200 },
+});
+
+// `body` is ephemeral: assertable at evaluation time, never persisted.
+const collectorResultSchema = z.object({
+  statusCode: healthResultNumber({
+    "x-chart-type": "counter",
+    "x-anomaly-enabled": false,
+  }),
+  body: healthResultString({ "x-ephemeral": true }),
+});
+
+function buildService({
+  entries,
+  inserted,
+}: {
+  entries: CollectorConfigEntry[];
+  inserted: Record<string, unknown>[];
+}) {
+  const tx = {
+    insert: mock(() => ({
+      values: mock((vals: Record<string, unknown>) => {
+        inserted.push(vals);
+        return Object.assign(Promise.resolve(), {
+          onConflictDoUpdate: mock(() => Promise.resolve()),
+          onConflictDoNothing: mock(() => Promise.resolve()),
+        });
+      }),
+    })),
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() =>
+          Object.assign(Promise.resolve([]), {
+            limit: mock(() => Promise.resolve([])),
+          }),
+        ),
+      })),
+    })),
+  };
+
+  const db = {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() => Promise.resolve([{ collectors: entries }])),
+      })),
+    })),
+    transaction: mock(async (fn: (t: typeof tx) => Promise<void>) => fn(tx)),
+  };
+
+  const collectorRegistry = {
+    register: mock(() => {}),
+    getCollector: mock(() => ({
+      collector: {
+        id: "test-collector",
+        result: new Versioned({ version: 1, schema: collectorResultSchema }),
+      },
+    })),
+    getCollectors: mock(() => []),
+  };
+
+  return new HealthCheckService(
+    db as unknown as ConstructorParameters<typeof HealthCheckService>[0],
+    {} as unknown as ConstructorParameters<typeof HealthCheckService>[1],
+    collectorRegistry as unknown as ConstructorParameters<
+      typeof HealthCheckService
+    >[2],
+  );
+}
+
+function satelliteResult({ statusCode }: { statusCode: number }) {
+  return {
+    status: "healthy",
+    latencyMs: 42,
+    message: "Completed in 42ms",
+    metadata: {
+      collectors: {
+        "entry-1": {
+          _collectorId: "test-collector",
+          statusCode,
+          body: '{"status":"ok"}',
+        },
+      },
+    },
+  };
+}
+
+const entries: CollectorConfigEntry[] = [
+  {
+    id: "entry-1",
+    collectorId: "test-collector",
+    config: {},
+    assertions: [{ field: "statusCode", operator: "equals", value: 200 }],
+  },
+];
+
+async function ingest({
+  entries,
+  statusCode,
+}: {
+  entries: CollectorConfigEntry[];
+  statusCode: number;
+}) {
+  const inserted: Record<string, unknown>[] = [];
+  const service = buildService({ entries, inserted });
+  await service.ingestSatelliteResult({
+    configId: "config-1",
+    systemId: "system-1",
+    status: "healthy",
+    latencyMs: 42,
+    result: satelliteResult({ statusCode }) as never,
+    executedAt: "2026-07-03T10:00:00.000Z",
+    sourceId: "sat-1",
+    sourceLabel: "EU West",
+  });
+  const runInsert = inserted.find((v) => "status" in v && "result" in v);
+  expect(runInsert).toBeDefined();
+  return runInsert as Record<string, unknown>;
+}
+
+function collectorEntryOf(runInsert: Record<string, unknown>) {
+  const result = runInsert.result as {
+    metadata: { collectors: Record<string, Record<string, unknown>> };
+  };
+  return result.metadata.collectors["entry-1"];
+}
+
+describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
+  it("downgrades a satellite-healthy run whose assertion fails", async () => {
+    const runInsert = await ingest({ entries, statusCode: 404 });
+    expect(runInsert.status).toBe("unhealthy");
+
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertionFailed).toBe("statusCode equals 200");
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ key: KEY, passed: false, actual: "404" }),
+    ]);
+    const message = (runInsert.result as { message: string }).message;
+    expect(message).toBe(
+      "Check failed: Assertion failed: statusCode equals 200",
+    );
+  });
+
+  it("keeps a passing run healthy and stores the passing outcome", async () => {
+    const runInsert = await ingest({ entries, statusCode: 200 });
+    expect(runInsert.status).toBe("healthy");
+
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertionFailed).toBeUndefined();
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ key: KEY, passed: true, actual: "200" }),
+    ]);
+  });
+
+  it("strips ephemeral fields AFTER assertions ran against them", async () => {
+    const withBodyAssertion: CollectorConfigEntry[] = [
+      {
+        id: "entry-1",
+        collectorId: "test-collector",
+        config: {},
+        assertions: [
+          {
+            field: "body.$",
+            jsonPath: "$.status",
+            operator: "equals",
+            value: "ok",
+          },
+        ],
+      },
+    ];
+    const runInsert = await ingest({
+      entries: withBodyAssertion,
+      statusCode: 200,
+    });
+
+    const entry = collectorEntryOf(runInsert);
+    // The JSONPath assertion evaluated against the (ephemeral) body...
+    expect(entry._assertions).toEqual([
+      expect.objectContaining({ passed: true, actual: "ok" }),
+    ]);
+    // ...but the body itself never reaches storage.
+    expect(entry.body).toBeUndefined();
+    expect(entry.statusCode).toBe(200);
+    expect(runInsert.status).toBe("healthy");
+  });
+
+  it("tolerates collector entries the config no longer knows", async () => {
+    const runInsert = await ingest({ entries: [], statusCode: 500 });
+    // No assertions configured: status passes through untouched.
+    expect(runInsert.status).toBe("healthy");
+    const entry = collectorEntryOf(runInsert);
+    expect(entry._assertions).toBeUndefined();
+  });
+});