@checkstack/healthcheck-backend 1.18.0 → 1.20.0

package/src/status-fingerprint.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "bun:test";
+import {
+  statusFingerprint,
+  statusVectorChanged,
+  type StatusFingerprintInput,
+} from "./status-fingerprint";
+
+const check = (
+  configurationId: string,
+  status: string,
+  sliceCount = 1,
+  failingSliceCount = 0,
+): StatusFingerprintInput["checkStatuses"][number] => ({
+  configurationId,
+  status,
+  sliceCount,
+  failingSliceCount,
+});
+
+describe("statusFingerprint", () => {
+  it("is invariant to check ORDER", () => {
+    const a: StatusFingerprintInput = {
+      status: "degraded",
+      checkStatuses: [check("c1", "healthy"), check("c2", "degraded")],
+    };
+    const b: StatusFingerprintInput = {
+      status: "degraded",
+      checkStatuses: [check("c2", "degraded"), check("c1", "healthy")],
+    };
+    expect(statusFingerprint(a)).toBe(statusFingerprint(b));
+  });
+
+  it("differs when a check's status flips", () => {
+    const before = statusFingerprint({
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy")],
+    });
+    const after = statusFingerprint({
+      status: "healthy",
+      checkStatuses: [check("c1", "degraded")],
+    });
+    expect(before).not.toBe(after);
+  });
+
+  it("differs when only the slice failure count changes", () => {
+    const before = statusFingerprint({
+      status: "degraded",
+      checkStatuses: [check("c1", "degraded", 3, 1)],
+    });
+    const after = statusFingerprint({
+      status: "degraded",
+      checkStatuses: [check("c1", "degraded", 3, 2)],
+    });
+    expect(before).not.toBe(after);
+  });
+});
+
+describe("statusVectorChanged", () => {
+  const base: StatusFingerprintInput = {
+    status: "healthy",
+    checkStatuses: [check("c1", "healthy"), check("c2", "healthy")],
+  };
+
+  it("is FALSE for a pure timestamp/runs refresh (same vector)", () => {
+    // Volatile fields (evaluatedAt / lastRunAt / runsConsidered) are not part of
+    // the fingerprint, so an object carrying different ones but the same vector
+    // is not a change. Represented here by an identical vector.
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy"), check("c2", "healthy")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(false);
+  });
+
+  it("is TRUE for a per-check flip even when the rollup enum is unchanged", () => {
+    // Rollup stays "healthy" here (contrived), but c2 flipped — the entity view
+    // {status, healthyChecks, totalChecks} could miss this, the fingerprint does not.
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy"), check("c2", "degraded")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(true);
+  });
+
+  it("is TRUE when the check SET changes", () => {
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(true);
+  });
+});

package/src/status-fingerprint.ts

@@ -0,0 +1,66 @@
+/**
+ * Per-check status fingerprint — the change-signal that gates system-health
+ * cache invalidation.
+ *
+ * The cached value is the full aggregated system-health response
+ * (`getSystemHealthStatus`), but most of its fields are VOLATILE: every run
+ * bumps `evaluatedAt`, a check's `lastRunAt`, and `runsConsidered` even when
+ * nothing an operator sees actually changed. Invalidating on those would defeat
+ * the cache (every tick evicts). The fingerprint is invariant to them: it
+ * captures ONLY the derived-status vector a reader gates on:
+ *  - the system-wide rollup `status`, and
+ *  - per check: `configurationId`, its derived `status`, and its slice
+ *    composition (`sliceCount` / `failingSliceCount`).
+ *
+ * So `statusVectorChanged(prev, next)` is true exactly when a check flipped
+ * status, a slice began/stopped failing, or the check set changed — i.e. when a
+ * status a reader renders actually moved. A run that merely refreshes timestamps
+ * with the same vector is NOT a change, so the reconcile skips invalidation and
+ * the cached value (correct except for volatile fields no reader needs) survives
+ * to its TTL. This is the "broaden the change-signal to the per-check status
+ * vector" requirement: it catches a per-check flip that leaves the rollup enum
+ * unchanged (which the entity view's `{status, healthyChecks, totalChecks}`
+ * would miss), while ignoring pure timestamp churn.
+ */
+
+/**
+ * Structural subset of `SystemHealthStatusResponse` the fingerprint reads. The
+ * service's response is assignable to this, so no import (and no cast) is needed
+ * — keeping this module a leaf the cache and executor can both depend on.
+ */
+export interface StatusFingerprintInput {
+  status: string;
+  checkStatuses: readonly {
+    configurationId: string;
+    status: string;
+    sliceCount: number;
+    failingSliceCount: number;
+  }[];
+}
+
+/**
+ * A stable, order-independent fingerprint of a system-health response's derived
+ * status vector. Checks are sorted by `configurationId` so two responses with
+ * the same checks in a different order fingerprint identically.
+ */
+export function statusFingerprint(response: StatusFingerprintInput): string {
+  const perCheck = response.checkStatuses
+    .map(
+      (c) =>
+        `${c.configurationId}:${c.status}:${c.sliceCount}:${c.failingSliceCount}`,
+    )
+    .toSorted();
+  return `${response.status}|${perCheck.join(",")}`;
+}
+
+/**
+ * Whether the derived status vector changed between two responses (ignoring
+ * volatile fields). This is the gate for cache invalidation + cross-pod
+ * broadcast: only a real vector change evicts the cache and wakes other pods.
+ */
+export function statusVectorChanged(
+  previous: StatusFingerprintInput,
+  next: StatusFingerprintInput,
+): boolean {
+  return statusFingerprint(previous) !== statusFingerprint(next);
+}

package/src/status-page/rollup.test.ts

@@ -3,6 +3,7 @@ import {
   mapHealthStatus,
   rollupStatus,
   overallBannerStatus,
+  rollupSelectedEnvironments,
   statusBannerTitle,
 } from "./rollup";
 
@@ -45,6 +46,45 @@ describe("overallBannerStatus", () => {
   });
 });
 
+describe("rollupSelectedEnvironments", () => {
+  const environments = {
+    prod: { status: "healthy" },
+    staging: { status: "unhealthy" },
+    dev: { status: "degraded" },
+  };
+  test("considers only the selected environments (worst-wins)", () => {
+    expect(
+      rollupSelectedEnvironments({
+        environments,
+        selectedEnvironmentIds: ["prod"],
+      }),
+    ).toBe("operational");
+    expect(
+      rollupSelectedEnvironments({
+        environments,
+        selectedEnvironmentIds: ["prod", "staging"],
+      }),
+    ).toBe("major_outage");
+    expect(
+      rollupSelectedEnvironments({
+        environments,
+        selectedEnvironmentIds: ["prod", "dev"],
+      }),
+    ).toBe("degraded");
+  });
+  test("no slice in any selected env -> unknown", () => {
+    expect(
+      rollupSelectedEnvironments({
+        environments,
+        selectedEnvironmentIds: ["nonexistent"],
+      }),
+    ).toBe("unknown");
+    expect(
+      rollupSelectedEnvironments({ environments, selectedEnvironmentIds: [] }),
+    ).toBe("unknown");
+  });
+});
+
 describe("statusBannerTitle", () => {
   test("renders a human title", () => {
     expect(statusBannerTitle("operational")).toBe("All systems operational");

package/src/status-page/rollup.ts

@@ -58,6 +58,33 @@ export function overallBannerStatus(statuses: PublicStatus[]): PublicStatus {
   return rollupStatus(known);
 }
 
+/**
+ * Roll up a system's PER-ENVIRONMENT health-check statuses into a single public
+ * status for a status page scoped to specific environments. ONLY the slices for
+ * the selected environments are considered (worst-status-wins via the public
+ * precedence); a system with no slice in any selected environment resolves to
+ * `unknown`. Used by the health widgets when a page publishes an explicit
+ * environment set, so a system in both prod and staging shows only its
+ * selected-environment health rather than the cross-environment rollup.
+ *
+ * This considers CHECK statuses only. Incident-forced overrides are whole-system
+ * (not environment-scoped), so the caller folds the override IN via worst-wins on
+ * top of this result (see `healthPublicStatuses`) - keeping this helper a pure
+ * per-environment checks rollup.
+ */
+export function rollupSelectedEnvironments(args: {
+  environments: Record<string, { status: string }>;
+  selectedEnvironmentIds: string[];
+}): PublicStatus {
+  const { environments, selectedEnvironmentIds } = args;
+  const statuses: PublicStatus[] = [];
+  for (const envId of selectedEnvironmentIds) {
+    const slice = environments[envId];
+    if (slice) statuses.push(mapHealthStatus(slice.status));
+  }
+  return rollupStatus(statuses);
+}
+
 export function statusBannerTitle(status: PublicStatus): string {
   switch (status) {
     case "operational": {

package/src/status-page/widgets.test.ts

@@ -0,0 +1,387 @@
+import { describe, test, expect } from "bun:test";
+import type { RpcClient } from "@checkstack/backend-api";
+import type {
+  WidgetResolveContext,
+  WidgetTypeDefinition,
+  StatusWidgetTypeExtensionPoint,
+} from "@checkstack/status-page-backend";
+import { registerHealthcheckStatusWidgets } from "./widgets";
+
+/**
+ * Widget-level regression coverage for status-page environment filtering (the
+ * gap that let the standalone uptime widget slip through). Exercises the four
+ * health widgets directly against a mocked trusted rpcClient, asserting that a
+ * published-environment set:
+ *  - omits systems outside the selected environments from banner / systemHealth
+ *    / groupStatus AND blanks the single-system uptime widget, and
+ *  - folds the WHOLE-SYSTEM incident override into the env-scoped health rollup
+ *    (a prod system whose prod checks are green but which is under an active
+ *    incident-forced outage still reads as an outage on a prod-scoped page).
+ */
+
+/** Per-system per-environment CHECKS status, as getBulkSystemHealthMatrix returns. */
+type MatrixEnvStatus = "healthy" | "degraded" | "unhealthy";
+
+interface MockData {
+  /** environmentId -> member system ids (catalog env->systems mapping). */
+  envSystems: Record<string, string[]>;
+  /** systemId -> display name. */
+  systemNames: Record<string, string>;
+  /** systemId -> per-environment CHECKS status. */
+  matrix: Record<string, Record<string, MatrixEnvStatus>>;
+  /** systemId -> whole-system incident override status (folded, worst-wins). */
+  overrides?: Record<string, MatrixEnvStatus>;
+  /** systemId -> total uptime percent (for the uptime widget). */
+  uptimePct?: Record<string, number>;
+  /** catalog groupId -> member system ids. */
+  groups?: Record<string, string[]>;
+}
+
+function widgetsById(): Map<string, WidgetTypeDefinition> {
+  const map = new Map<string, WidgetTypeDefinition>();
+  const ext: StatusWidgetTypeExtensionPoint = {
+    registerWidgetType: (def) => map.set(def.id, def),
+  };
+  registerHealthcheckStatusWidgets(ext);
+  return map;
+}
+
+function makeCtx(args: {
+  data: MockData;
+  publishedEnvironmentIds?: string[];
+}): WidgetResolveContext {
+  const { data, publishedEnvironmentIds } = args;
+  const memo = new Map<string, Promise<unknown>>();
+  const api = {
+    // catalog
+    resolveEnvironments: async ({
+      environmentIds,
+    }: {
+      environmentIds: string[];
+    }) =>
+      environmentIds.map((id) => ({
+        id,
+        name: id,
+        description: null,
+        systemIds: data.envSystems[id] ?? [],
+        metadata: null,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      })),
+    getSystems: async () => ({
+      systems: Object.entries(data.systemNames).map(([id, name]) => ({
+        id,
+        name,
+      })),
+    }),
+    getGroups: async () =>
+      Object.entries(data.groups ?? {}).map(([id, systemIds]) => ({
+        id,
+        name: id,
+        systemIds,
+      })),
+    // maintenance
+    getBulkMaintenancesForSystems: async () => ({ maintenances: {} }),
+    // healthcheck
+    getBulkSystemHealthMatrix: async ({ systemIds }: { systemIds: string[] }) => {
+      const statuses: Record<
+        string,
+        {
+          status: MatrixEnvStatus;
+          checkStatuses: never[];
+          environments: Record<
+            string,
+            { status: MatrixEnvStatus; checkStatuses: never[] }
+          >;
+        }
+      > = {};
+      for (const id of systemIds) {
+        const envs = data.matrix[id];
+        if (!envs) continue;
+        statuses[id] = {
+          status: "healthy",
+          checkStatuses: [],
+          environments: Object.fromEntries(
+            Object.entries(envs).map(([env, status]) => [
+              env,
+              { status, checkStatuses: [] },
+            ]),
+          ),
+        };
+      }
+      return { statuses };
+    },
+    getBulkSystemHealthStatus: async ({ systemIds }: { systemIds: string[] }) => {
+      const statuses: Record<
+        string,
+        {
+          status: MatrixEnvStatus;
+          evaluatedAt: Date;
+          checkStatuses: never[];
+          override?: { status: MatrixEnvStatus; source: string; reason: string };
+        }
+      > = {};
+      for (const id of systemIds) {
+        const override = data.overrides?.[id];
+        statuses[id] = {
+          status: "healthy",
+          evaluatedAt: new Date(),
+          checkStatuses: [],
+          ...(override
+            ? { override: { status: override, source: "incident", reason: "x" } }
+            : {}),
+        };
+      }
+      return { statuses };
+    },
+    getRunStats: async ({ systemId }: { systemId: string }) => {
+      const pct = data.uptimePct?.[systemId] ?? 100;
+      return {
+        window: { start: "2026-07-01T00:00:00Z", end: "2026-07-02T00:00:00Z" },
+        bucketIntervalSeconds: 86_400,
+        total: {
+          runCount: 10,
+          healthy: 10,
+          degraded: 0,
+          unhealthy: 0,
+          uptimePct: pct,
+        },
+        buckets: [
+          {
+            start: "2026-07-01T00:00:00Z",
+            end: "2026-07-02T00:00:00Z",
+            runCount: 10,
+            healthy: 10,
+            degraded: 0,
+            unhealthy: 0,
+            uptimePct: pct,
+          },
+        ],
+      };
+    },
+  };
+  return {
+    rpcClient: { forPlugin: () => api } as unknown as RpcClient,
+    cache: <T,>(key: string, loader: () => Promise<T>): Promise<T> => {
+      const existing = memo.get(key);
+      if (existing) return existing as Promise<T>;
+      const created = loader();
+      memo.set(key, created);
+      return created;
+    },
+    publishedEnvironmentIds,
+  };
+}
+
+// prod-sys: prod only (checks healthy, but under an active incident override).
+// stage-sys: staging only (checks unhealthy).
+// both-sys: prod + staging (prod healthy, staging unhealthy).
+const DATA: MockData = {
+  envSystems: {
+    prod: ["prod-sys", "both-sys"],
+    stage: ["stage-sys", "both-sys"],
+  },
+  systemNames: {
+    "prod-sys": "Prod System",
+    "stage-sys": "Stage System",
+    "both-sys": "Both System",
+  },
+  matrix: {
+    "prod-sys": { prod: "healthy" },
+    "stage-sys": { stage: "unhealthy" },
+    "both-sys": { prod: "healthy", stage: "unhealthy" },
+  },
+  overrides: { "prod-sys": "unhealthy" }, // whole-system incident-forced outage
+  uptimePct: { "prod-sys": 99, "stage-sys": 50, "both-sys": 88 },
+  groups: { g1: ["prod-sys", "stage-sys"] },
+};
+
+describe("health widgets — environment filtering (E3 regression)", () => {
+  const publishedEnvironmentIds = ["prod"];
+
+  test("systemHealth omits staging-only systems and folds the whole-system override", async () => {
+    const widget = widgetsById().get("systemHealth")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds });
+    const result = (await widget.resolvePublic({
+      config: {
+        items: [
+          { systemId: "prod-sys" },
+          { systemId: "stage-sys" },
+          { systemId: "both-sys" },
+        ],
+      },
+      ctx,
+    })) as { systems: Array<{ label: string; status: string }> };
+    // stage-sys is dropped; prod-sys shows the incident override (major_outage)
+    // even though its prod checks are healthy; both-sys shows its prod checks.
+    expect(result.systems).toEqual([
+      { label: "Prod System", status: "major_outage" },
+      { label: "Both System", status: "operational" },
+    ]);
+  });
+
+  test("banner rolls up only env-visible systems, override included", async () => {
+    const widget = widgetsById().get("banner")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds });
+    const result = (await widget.resolvePublic({
+      config: { systemIds: ["prod-sys", "stage-sys", "both-sys"] },
+      ctx,
+    })) as { status: string };
+    // Visible = {prod-sys (major via override), both-sys (operational)} ->
+    // some-but-not-all down = partial_outage. stage-sys never counted.
+    expect(result.status).toBe("partial_outage");
+  });
+
+  test("groupStatus drops group members outside the published environments", async () => {
+    const widget = widgetsById().get("groupStatus")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds });
+    const result = (await widget.resolvePublic({
+      config: { groupId: "g1" },
+      ctx,
+    })) as { systems: Array<{ label: string; status: string }>; status: string };
+    // g1 = [prod-sys, stage-sys]; only prod-sys is in prod. Its override lifts
+    // it to major_outage; stage-sys is omitted entirely.
+    expect(result.systems).toEqual([
+      { label: "Prod System", status: "major_outage" },
+    ]);
+    expect(result.status).toBe("major_outage");
+  });
+
+  test("uptime widget is blanked for a system outside the published environments", async () => {
+    const widget = widgetsById().get("uptime")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds });
+    const result = (await widget.resolvePublic({
+      config: { systemId: "stage-sys", days: 30 },
+      ctx,
+    })) as { label: string; uptimePct: number; bars: unknown[] };
+    // Out of scope -> blank empty-state DTO (no bars, no misleading percent).
+    expect(result.bars).toEqual([]);
+    expect(result.uptimePct).toBe(0);
+  });
+
+  test("uptime widget renders normally for an in-scope system", async () => {
+    const widget = widgetsById().get("uptime")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds });
+    const result = (await widget.resolvePublic({
+      config: { systemId: "prod-sys", days: 30 },
+      ctx,
+    })) as { label: string; uptimePct: number; bars: unknown[] };
+    expect(result.label).toBe("Prod System");
+    expect(result.bars.length).toBe(1);
+    expect(result.uptimePct).toBe(99);
+  });
+});
+
+describe("health widgets — no environment filter (unchanged behavior)", () => {
+  test("systemHealth keeps every system and uses the folded cross-env status", async () => {
+    const widget = widgetsById().get("systemHealth")!;
+    const ctx = makeCtx({ data: DATA }); // no publishedEnvironmentIds
+    const result = (await widget.resolvePublic({
+      config: {
+        items: [{ systemId: "prod-sys" }, { systemId: "stage-sys" }],
+      },
+      ctx,
+    })) as { systems: Array<{ label: string; status: string }> };
+    // All-env path reads getBulkSystemHealthStatus.status (healthy in the mock)
+    // for both systems; no system is dropped.
+    expect(result.systems).toEqual([
+      { label: "Prod System", status: "operational" },
+      { label: "Stage System", status: "operational" },
+    ]);
+  });
+
+  test("uptime widget is never blanked when no environment filter is set", async () => {
+    const widget = widgetsById().get("uptime")!;
+    const ctx = makeCtx({ data: DATA });
+    const result = (await widget.resolvePublic({
+      config: { systemId: "stage-sys", days: 30 },
+      ctx,
+    })) as { bars: unknown[]; uptimePct: number };
+    expect(result.bars.length).toBe(1);
+    expect(result.uptimePct).toBe(50);
+  });
+});
+
+/**
+ * Send-time SCOPING regression: the subscriber fan-out only surfaces a HEALTH
+ * notification through a health widget, and only for the systems the widget
+ * CURRENTLY shows (its configured systems ∩ the page's published environments).
+ * Each health widget must therefore declare `subscriptionCategory: "health"` and
+ * a `resolveScopedSystems` that matches what `resolvePublic` renders.
+ */
+describe("health widgets — resolveScopedSystems (send-time scoping)", () => {
+  test("every health widget is tagged with the 'health' subscription category", () => {
+    const widgets = widgetsById();
+    for (const id of ["banner", "systemHealth", "groupStatus", "uptime"]) {
+      expect(widgets.get(id)!.subscriptionCategory).toBe("health");
+    }
+  });
+
+  const scoped = ["prod"];
+
+  test("banner scopes its systems to the published environments", async () => {
+    const widget = widgetsById().get("banner")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const set = await widget.resolveScopedSystems!({
+      config: { systemIds: ["prod-sys", "stage-sys", "both-sys"] },
+      ctx,
+    });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys"]);
+  });
+
+  test("systemHealth scopes to env and applies per-row label overrides in the detailed list", async () => {
+    const widget = widgetsById().get("systemHealth")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const config = {
+      items: [
+        { systemId: "prod-sys", label: "PROD!" },
+        { systemId: "stage-sys" },
+        { systemId: "both-sys" },
+      ],
+    };
+    const set = await widget.resolveScopedSystems!({ config, ctx });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys"]);
+    const detailed = await widget.resolveScopedSystemsDetailed!({ config, ctx });
+    expect(detailed).toEqual([
+      { id: "prod-sys", name: "PROD!" }, // label override wins over catalog name
+      { id: "both-sys", name: "Both System" },
+    ]);
+  });
+
+  test("groupStatus scopes its expanded members to the published environments", async () => {
+    const widget = widgetsById().get("groupStatus")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const set = await widget.resolveScopedSystems!({
+      config: { groupId: "g1" },
+      ctx,
+    });
+    expect([...set]).toEqual(["prod-sys"]); // g1 = [prod-sys, stage-sys]; prod only
+  });
+
+  test("uptime scopes to its single system, emptying when it is out of scope", async () => {
+    const widget = widgetsById().get("uptime")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    expect([
+      ...(await widget.resolveScopedSystems!({
+        config: { systemId: "prod-sys", days: 30 },
+        ctx,
+      })),
+    ]).toEqual(["prod-sys"]);
+    expect([
+      ...(await widget.resolveScopedSystems!({
+        config: { systemId: "stage-sys", days: 30 },
+        ctx,
+      })),
+    ]).toEqual([]);
+  });
+
+  test("no environment filter surfaces every configured system", async () => {
+    const widget = widgetsById().get("banner")!;
+    const ctx = makeCtx({ data: DATA }); // all environments
+    const set = await widget.resolveScopedSystems!({
+      config: { systemIds: ["prod-sys", "stage-sys", "both-sys"] },
+      ctx,
+    });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys", "stage-sys"]);
+  });
+});