@checkstack/healthcheck-backend 1.15.0 → 1.17.0

package/src/service.ts

@@ -12,7 +12,9 @@ import {
   type CollectorConfigEntry,
   type HealthcheckSignalStatuses,
   type RunStats,
+  stripEphemeralFields,
 } from "@checkstack/healthcheck-common";
+import { evaluateCollectorAssertionOutcomes } from "./collector-assertions";
 import { summarizeRuns, type StatRun } from "./run-stats.logic";
 import type { ConfigService } from "@checkstack/backend-api";
 import type { InferClient } from "@checkstack/common";
@@ -39,6 +41,7 @@ import {
   gte,
   lte,
   isNull,
+  isNotNull,
   inArray,
 } from "drizzle-orm";
 import { ORPCError } from "@orpc/server";
@@ -57,12 +60,14 @@ import type {
 import {
   aggregateCollectorData,
   extractLatencies,
+  foldRunAssertionStats,
   mergeTieredBuckets,
   reaggregateBuckets,
   countStatuses,
   calculateLatencyStats,
   type NormalizedBucket,
 } from "./aggregation-utils";
+import { ASSERTIONS_AGG_KEY } from "@checkstack/healthcheck-common";
 import {
   extractConfigurationSecrets,
   mergeConfigurationSecrets,
@@ -1337,6 +1342,98 @@ export class HealthCheckService {
     return Object.fromEntries(entries);
   }
 
+  /**
+   * Bulk per-(system, check, environment) health for the given systems.
+   *
+   * For each system returns the cross-environment rollup (status +
+   * checkStatuses, same as {@link getSystemHealthStatus}) PLUS a slice per
+   * environment the system has runs for. Consumers that scope by environment
+   * (the dependency map) must read the per-environment slice, because the
+   * rollup deliberately hides a single failing environment.
+   *
+   * Cost scales with the number of environments each system actually fans out
+   * to (`1 + #envs` status evaluations per system); systems with only env-less
+   * runs cost the same as a plain rollup read. Not on any per-run hot path.
+   */
+  async getBulkSystemHealthMatrix(systemIds: string[]): Promise<
+    Record<
+      string,
+      {
+        status: HealthCheckStatus;
+        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+        environments: Record<
+          string,
+          {
+            status: HealthCheckStatus;
+            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+          }
+        >;
+      }
+    >
+  > {
+    const result: Record<
+      string,
+      {
+        status: HealthCheckStatus;
+        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+        environments: Record<
+          string,
+          {
+            status: HealthCheckStatus;
+            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+          }
+        >;
+      }
+    > = {};
+
+    await Promise.all(
+      systemIds.map(async (systemId) => {
+        const overall = await this.getSystemHealthStatus(systemId);
+
+        // Environments this system actually has runs for (env-less excluded -
+        // it is folded into the rollup and never a real environment id).
+        const envRows = await this.db
+          .selectDistinct({ environmentId: healthCheckRuns.environmentId })
+          .from(healthCheckRuns)
+          .where(
+            and(
+              eq(healthCheckRuns.systemId, systemId),
+              isNotNull(healthCheckRuns.environmentId),
+            ),
+          );
+
+        const environments: Record<
+          string,
+          {
+            status: HealthCheckStatus;
+            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+          }
+        > = {};
+        await Promise.all(
+          envRows.map(async ({ environmentId }) => {
+            if (!environmentId) return;
+            const envStatus = await this.getSystemHealthStatus(
+              systemId,
+              environmentId,
+            );
+            environments[environmentId] = {
+              status: envStatus.status,
+              checkStatuses: envStatus.checkStatuses,
+            };
+          }),
+        );
+
+        result[systemId] = {
+          status: overall.status,
+          checkStatuses: overall.checkStatuses,
+          environments,
+        };
+      }),
+    );
+
+    return result;
+  }
+
   /**
    * Get comprehensive health overview for a system.
    * Returns all health checks with their last 25 runs for sparkline visualization.
@@ -2151,9 +2248,16 @@ export class HealthCheckService {
           );
         }
 
+        // Per-assertion pass/fail counts (platform-owned, sibling of
+        // `collectors` — see assertion-analytics in healthcheck-common).
+        const assertionStats = foldRunAssertionStats(bucket.runs);
+
         aggregatedResult = {
           ...strategyResult,
           ...(collectorsAggregated ? { collectors: collectorsAggregated } : {}),
+          ...(assertionStats === undefined
+            ? {}
+            : { [ASSERTIONS_AGG_KEY]: assertionStats }),
         };
       }
 
@@ -2421,6 +2525,15 @@ export class HealthCheckService {
    * Ingest a health check result from a satellite.
    * Stores the run with source attribution (sourceId + sourceLabel)
    * and triggers incremental aggregation to keep charts/availability current.
+   *
+   * Assertions are evaluated HERE, on the core, not on the satellite: the
+   * satellite never held the assertion semantics, so historically
+   * satellite-executed checks silently skipped assertions entirely.
+   * Evaluating at ingest fixes that for every satellite version with no
+   * wire-protocol change. Caveat: buffered results are evaluated against the
+   * configuration CURRENT at ingest time. Ephemeral result fields (e.g. raw
+   * HTTP bodies) are needed for JSONPath assertions and are stripped right
+   * after evaluation, matching what the local executor stores.
    */
   async ingestSatelliteResult(props: {
     configId: string;
@@ -2432,20 +2545,77 @@ export class HealthCheckService {
     sourceId: string;
     sourceLabel: string;
   }) {
-    const {
-      configId,
-      systemId,
-      status,
-      latencyMs,
-      result,
-      sourceId,
-      sourceLabel,
-    } = props;
+    const { configId, systemId, latencyMs, result, sourceId, sourceLabel } =
+      props;
 
     const resultRecord = result
       ? ({ ...result } as Record<string, unknown>)
       : {};
 
+    let status = props.status;
+    const metadata = resultRecord.metadata as
+      | Record<string, unknown>
+      | undefined;
+    const collectorsMeta = metadata?.collectors as
+      | Record<string, Record<string, unknown>>
+      | undefined;
+    if (collectorsMeta && Object.keys(collectorsMeta).length > 0) {
+      const [configRow] = await this.db
+        .select({ collectors: healthCheckConfigurations.collectors })
+        .from(healthCheckConfigurations)
+        .where(eq(healthCheckConfigurations.id, configId));
+      const entries: CollectorConfigEntry[] = configRow?.collectors ?? [];
+
+      let firstFailure: string | undefined;
+      const nextCollectorsMeta: Record<string, Record<string, unknown>> = {
+        ...collectorsMeta,
+      };
+      for (const entry of entries) {
+        const entryResult = nextCollectorsMeta[entry.id];
+        if (!entryResult || typeof entryResult !== "object") continue;
+
+        let evaluated: Record<string, unknown> = { ...entryResult };
+        if (entry.assertions?.length) {
+          const evaluation = evaluateCollectorAssertionOutcomes({
+            assertions: entry.assertions,
+            result: evaluated,
+          });
+          evaluated._assertions = evaluation.outcomes;
+          evaluated._assertionFailed = evaluation.firstFailureMessage;
+          if (
+            evaluation.firstFailureMessage !== undefined &&
+            firstFailure === undefined
+          ) {
+            firstFailure = evaluation.firstFailureMessage;
+          }
+        }
+
+        // Parity with the local executor: satellites send raw results, so
+        // ephemeral fields (assertable but never persisted) get stripped
+        // here, AFTER assertions ran against them.
+        const registered = this.collectorRegistry.getCollector(
+          entry.collectorId,
+        );
+        if (registered) {
+          evaluated = stripEphemeralFields(
+            evaluated,
+            registered.collector.result.schema,
+          );
+        }
+        nextCollectorsMeta[entry.id] = evaluated;
+      }
+
+      resultRecord.metadata = { ...metadata, collectors: nextCollectorsMeta };
+
+      // Mirror the local executor: a failed assertion downgrades a run the
+      // satellite reported healthy.
+      if (firstFailure !== undefined && status === "healthy") {
+        status = "unhealthy";
+        resultRecord.status = status;
+        resultRecord.message = `Check failed: Assertion failed: ${firstFailure}`;
+      }
+    }
+
     // Atomic: the run row and the hourly-aggregate increment it feeds must
     // commit together. Without the transaction a failure on the (non-idempotent
     // `runCount + 1`) aggregate left a committed run that the aggregate never

package/src/status-page/widgets.ts

@@ -91,7 +91,17 @@ async function healthStatuses(
   const { statuses } = await ctx.rpcClient
     .forPlugin(HealthCheckApi)
     .getBulkSystemHealthStatus({ systemIds: ids });
-  return statuses;
+  // Project to ONLY the derived status. `getBulkSystemHealthStatus` folds active
+  // incident overrides into `status` (so the public page shows the forced
+  // status), but the response also carries `override.reason` = the incident
+  // TITLE. Status pages are public and incidents may be hidden, so we drop
+  // everything but the status here - the incident name must never reach a public
+  // widget DTO.
+  const projected: Record<string, { status: string } | undefined> = {};
+  for (const [systemId, value] of Object.entries(statuses)) {
+    projected[systemId] = value ? { status: value.status } : undefined;
+  }
+  return projected;
 }
 
 function publicStatus({

package/src/system-health-override.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, it } from "bun:test";
+import {
+  applySystemHealthOverrides,
+  worstHealthStatus,
+  type SystemHealthOverrideInput,
+} from "./system-health-override";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
+
+const at = new Date("2026-07-05T00:00:00.000Z");
+
+function base(
+  status: SystemHealthStatusResponse["status"],
+): SystemHealthStatusResponse {
+  return { status, evaluatedAt: at, checkStatuses: [] };
+}
+
+function override(
+  status: "degraded" | "unhealthy",
+  sourceId: string,
+): SystemHealthOverrideInput {
+  return {
+    status,
+    source: "incident",
+    reason: `Incident ${sourceId}`,
+    sourceId,
+  };
+}
+
+describe("worstHealthStatus", () => {
+  it("orders unhealthy > degraded > healthy", () => {
+    expect(worstHealthStatus("healthy", "degraded")).toBe("degraded");
+    expect(worstHealthStatus("degraded", "unhealthy")).toBe("unhealthy");
+    expect(worstHealthStatus("unhealthy", "degraded")).toBe("unhealthy");
+    expect(worstHealthStatus("healthy", "healthy")).toBe("healthy");
+  });
+});
+
+describe("applySystemHealthOverrides", () => {
+  it("returns the base unchanged (no override field) when there are no overrides", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [],
+    });
+    expect(result.status).toBe("healthy");
+    expect(result.override).toBeUndefined();
+  });
+
+  it("raises a healthy system to the override status and records the source", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [override("unhealthy", "inc-1")],
+    });
+    expect(result.status).toBe("unhealthy");
+    expect(result.override).toEqual({
+      status: "unhealthy",
+      source: "incident",
+      reason: "Incident inc-1",
+      sourceId: "inc-1",
+    });
+  });
+
+  it("applies an override even when the system has no health checks", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"), // no-checks default is healthy
+      overrides: [override("degraded", "inc-2")],
+    });
+    expect(result.status).toBe("degraded");
+    expect(result.override?.status).toBe("degraded");
+  });
+
+  it("keeps the worse HEALTH CHECK status when a check is worse than the override", () => {
+    // Override says degraded, but a check reports unhealthy -> unhealthy wins.
+    const result = applySystemHealthOverrides({
+      base: base("unhealthy"),
+      overrides: [override("degraded", "inc-3")],
+    });
+    expect(result.status).toBe("unhealthy");
+    // The override is still surfaced (it contributed), at its own status.
+    expect(result.override?.status).toBe("degraded");
+  });
+
+  it("picks the worst override when several incidents override the same system", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [
+        override("degraded", "inc-a"),
+        override("unhealthy", "inc-b"),
+        override("degraded", "inc-c"),
+      ],
+    });
+    expect(result.status).toBe("unhealthy");
+    expect(result.override?.sourceId).toBe("inc-b");
+  });
+});

package/src/system-health-override.ts

@@ -0,0 +1,93 @@
+import type {
+  HealthCheckStatus,
+  SystemHealthStatusResponse,
+  SystemHealthOverride,
+} from "@checkstack/healthcheck-common";
+
+/**
+ * Worst-wins ordering for the derived health vocabulary: a higher rank is a
+ * worse status. Mirrors the inline `unhealthy > degraded > healthy` ordering the
+ * per-check rollup uses in `service.ts`, extracted so the incident-override fold
+ * shares the exact same comparison.
+ */
+const HEALTH_RANK: Record<HealthCheckStatus, number> = {
+  healthy: 0,
+  degraded: 1,
+  unhealthy: 2,
+};
+
+/** Returns whichever status is worse (ties return `a`). */
+export function worstHealthStatus(
+  a: HealthCheckStatus,
+  b: HealthCheckStatus,
+): HealthCheckStatus {
+  return HEALTH_RANK[b] > HEALTH_RANK[a] ? b : a;
+}
+
+/**
+ * A non-health-check contribution to a system's health, as consumed by the
+ * fold. Kept source-agnostic (`source`/`sourceId`) so the health plugin does not
+ * hard-code incident semantics; the incident-backed reader maps its rows into
+ * this shape.
+ */
+export interface SystemHealthOverrideInput {
+  /** The status this contributor forces. Never `healthy` in practice. */
+  status: HealthCheckStatus;
+  /** Contributor kind, e.g. "incident". */
+  source: string;
+  /** Human-readable reason, e.g. the incident title. */
+  reason: string;
+  /** Opaque id of the contributing record, e.g. the incident id. */
+  sourceId?: string;
+}
+
+/**
+ * Reads active health overrides for a set of systems. Implemented in the plugin
+ * wiring over the incident RPC; injected into the health service so the service
+ * stays free of a direct incident dependency and tests can stub it.
+ */
+export interface SystemHealthOverrideReader {
+  getActiveOverrides(
+    systemIds: string[],
+  ): Promise<Record<string, SystemHealthOverrideInput[]>>;
+}
+
+/**
+ * Fold a system's active overrides into its health-check-derived status via
+ * worst-wins. The overall status becomes the worst of the checks-derived status
+ * and every override, so an override that raises a system to `degraded` never
+ * masks a health check reporting `unhealthy` (the worse status always wins), and
+ * an override applies even when the system has no health checks at all.
+ *
+ * The worst contributing override (if any) is surfaced on `override` so a UI can
+ * explain why a system reads worse than its checks alone. When the overrides are
+ * empty the base response is returned unchanged (no `override`).
+ */
+export function applySystemHealthOverrides({
+  base,
+  overrides,
+}: {
+  base: SystemHealthStatusResponse;
+  overrides: SystemHealthOverrideInput[];
+}): SystemHealthStatusResponse {
+  if (overrides.length === 0) return base;
+
+  // The override that reads worst wins the `override` slot; a tie keeps the
+  // first (query order), which for incidents is a stable, arbitrary pick.
+  let worst = overrides[0]!;
+  for (const candidate of overrides) {
+    if (HEALTH_RANK[candidate.status] > HEALTH_RANK[worst.status]) {
+      worst = candidate;
+    }
+  }
+
+  const status = worstHealthStatus(base.status, worst.status);
+  const override: SystemHealthOverride = {
+    status: worst.status,
+    source: worst.source,
+    reason: worst.reason,
+    sourceId: worst.sourceId,
+  };
+
+  return { ...base, status, override };
+}