@checkstack/healthcheck-backend 1.19.0 → 1.20.0

package/src/service.ts

@@ -1166,8 +1166,9 @@ export class HealthCheckService {
 
       // Environment filter for the per-check run window. `undefined` (rollup)
       // adds no predicate; `null` filters to the env-less slice; a string
-      // filters to that environment. The lookup index leads with
-      // (system_id, environment_id, …) so the env-scoped query is index-efficient.
+      // filters to that environment. `health_check_runs_slice_recent_idx`
+      // (system_id, configuration_id, environment_id, timestamp) serves the
+      // env-scoped ORDER BY timestamp DESC LIMIT read as an index range scan.
       //
       // For the rollup, we deliberately do NOT apply a single envFilter to one
       // flat run list — see the per-association branch below for why.
@@ -1357,17 +1358,10 @@ export class HealthCheckService {
    * list and no process-local state.
    */
   async getAllUnhealthySystemStatuses(): Promise<HealthcheckSignalStatuses> {
-    // Distinct systemIds that have at least one ENABLED check association.
-    // `getSystemHealthStatus` already short-circuits to healthy for systems
-    // with no enabled associations, so this is the complete candidate set.
-    const rows = await this.db
-      .selectDistinct({ systemId: systemHealthChecks.systemId })
-      .from(systemHealthChecks)
-      .where(eq(systemHealthChecks.enabled, true));
-
+    const systemIds = await this.getUnhealthyCandidateSystemIds();
     const result: HealthcheckSignalStatuses = {};
     await Promise.all(
-      rows.map(async ({ systemId }) => {
+      systemIds.map(async (systemId) => {
         const status = await this.getSystemHealthStatus(systemId);
         if (status.status === "healthy") return; // problems only
         result[systemId] = status;
@@ -1376,6 +1370,24 @@ export class HealthCheckService {
     return result;
   }
 
+  /**
+   * The candidate set for a global problem scan: distinct systemIds that have at
+   * least one ENABLED check association. `getSystemHealthStatus` already
+   * short-circuits to healthy for systems with no enabled associations, so this
+   * is the complete set of systems that could be degraded/unhealthy. Split out
+   * so callers can route the per-system status reads through the SHARED CACHE
+   * (`HealthCheckCache.readBulk`) instead of the uncached N+1 that
+   * {@link getAllUnhealthySystemStatuses} runs. Derives only from the durable
+   * `system_health_checks` table, so it answers identically on every pod.
+   */
+  async getUnhealthyCandidateSystemIds(): Promise<string[]> {
+    const rows = await this.db
+      .selectDistinct({ systemId: systemHealthChecks.systemId })
+      .from(systemHealthChecks)
+      .where(eq(systemHealthChecks.enabled, true));
+    return rows.map((r) => r.systemId);
+  }
+
   /**
    * Live health-state snapshot for a single system (Wave-2 sensing
    * contract). When `configurationId` is given, status reflects that
@@ -1485,95 +1497,26 @@ export class HealthCheckService {
   }
 
   /**
-   * Bulk per-(system, check, environment) health for the given systems.
-   *
-   * For each system returns the cross-environment rollup (status +
-   * checkStatuses, same as {@link getSystemHealthStatus}) PLUS a slice per
-   * environment the system has runs for. Consumers that scope by environment
-   * (the dependency map) must read the per-environment slice, because the
-   * rollup deliberately hides a single failing environment.
-   *
-   * Cost scales with the number of environments each system actually fans out
-   * to (`1 + #envs` status evaluations per system); systems with only env-less
-   * runs cost the same as a plain rollup read. Not on any per-run hot path.
+   * Distinct environment ids a system currently has runs for (the env-less
+   * slice is excluded — it is folded into the rollup and is never a real
+   * environment id). Used by the status cache's `readMatrix` to enumerate which
+   * per-environment slices to read (each slice is then served from the same
+   * per-env cache the badge path warms), so the bulk matrix no longer fans out
+   * uncached `getSystemHealthStatus` calls of its own.
    */
-  async getBulkSystemHealthMatrix(systemIds: string[]): Promise<
-    Record<
-      string,
-      {
-        status: HealthCheckStatus;
-        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
-        environments: Record<
-          string,
-          {
-            status: HealthCheckStatus;
-            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
-          }
-        >;
-      }
-    >
-  > {
-    const result: Record<
-      string,
-      {
-        status: HealthCheckStatus;
-        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
-        environments: Record<
-          string,
-          {
-            status: HealthCheckStatus;
-            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
-          }
-        >;
-      }
-    > = {};
-
-    await Promise.all(
-      systemIds.map(async (systemId) => {
-        const overall = await this.getSystemHealthStatus(systemId);
-
-        // Environments this system actually has runs for (env-less excluded -
-        // it is folded into the rollup and never a real environment id).
-        const envRows = await this.db
-          .selectDistinct({ environmentId: healthCheckRuns.environmentId })
-          .from(healthCheckRuns)
-          .where(
-            and(
-              eq(healthCheckRuns.systemId, systemId),
-              isNotNull(healthCheckRuns.environmentId),
-            ),
-          );
-
-        const environments: Record<
-          string,
-          {
-            status: HealthCheckStatus;
-            checkStatuses: SystemHealthStatusResponse["checkStatuses"];
-          }
-        > = {};
-        await Promise.all(
-          envRows.map(async ({ environmentId }) => {
-            if (!environmentId) return;
-            const envStatus = await this.getSystemHealthStatus(
-              systemId,
-              environmentId,
-            );
-            environments[environmentId] = {
-              status: envStatus.status,
-              checkStatuses: envStatus.checkStatuses,
-            };
-          }),
-        );
-
-        result[systemId] = {
-          status: overall.status,
-          checkStatuses: overall.checkStatuses,
-          environments,
-        };
-      }),
-    );
-
-    return result;
+  async getSystemEnvironmentIds(systemId: string): Promise<string[]> {
+    const envRows = await this.db
+      .selectDistinct({ environmentId: healthCheckRuns.environmentId })
+      .from(healthCheckRuns)
+      .where(
+        and(
+          eq(healthCheckRuns.systemId, systemId),
+          isNotNull(healthCheckRuns.environmentId),
+        ),
+      );
+    return envRows
+      .map((r) => r.environmentId)
+      .filter((id): id is string => id !== null);
   }
 
   /**
@@ -1648,9 +1591,9 @@ export class HealthCheckService {
         // Most recent HEALTHY run per environment, computed OUTSIDE the bounded
         // sparkline window so "last successful run" stays correct even when a
         // check has been failing for far longer than the last 25 runs. One
-        // grouped aggregate query per check (env-less = the `null` group). The
-        // (system_id, configuration_id, environment_id, timestamp) index makes
-        // this a cheap max-per-group scan.
+        // grouped aggregate query per check (env-less = the `null` group).
+        // `health_check_runs_slice_recent_idx` (system_id, configuration_id,
+        // environment_id, timestamp) makes this a cheap max-per-group scan.
         const lastHealthyRows = await tx
           .select({
             environmentId: healthCheckRuns.environmentId,

package/src/status-fingerprint.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "bun:test";
+import {
+  statusFingerprint,
+  statusVectorChanged,
+  type StatusFingerprintInput,
+} from "./status-fingerprint";
+
+const check = (
+  configurationId: string,
+  status: string,
+  sliceCount = 1,
+  failingSliceCount = 0,
+): StatusFingerprintInput["checkStatuses"][number] => ({
+  configurationId,
+  status,
+  sliceCount,
+  failingSliceCount,
+});
+
+describe("statusFingerprint", () => {
+  it("is invariant to check ORDER", () => {
+    const a: StatusFingerprintInput = {
+      status: "degraded",
+      checkStatuses: [check("c1", "healthy"), check("c2", "degraded")],
+    };
+    const b: StatusFingerprintInput = {
+      status: "degraded",
+      checkStatuses: [check("c2", "degraded"), check("c1", "healthy")],
+    };
+    expect(statusFingerprint(a)).toBe(statusFingerprint(b));
+  });
+
+  it("differs when a check's status flips", () => {
+    const before = statusFingerprint({
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy")],
+    });
+    const after = statusFingerprint({
+      status: "healthy",
+      checkStatuses: [check("c1", "degraded")],
+    });
+    expect(before).not.toBe(after);
+  });
+
+  it("differs when only the slice failure count changes", () => {
+    const before = statusFingerprint({
+      status: "degraded",
+      checkStatuses: [check("c1", "degraded", 3, 1)],
+    });
+    const after = statusFingerprint({
+      status: "degraded",
+      checkStatuses: [check("c1", "degraded", 3, 2)],
+    });
+    expect(before).not.toBe(after);
+  });
+});
+
+describe("statusVectorChanged", () => {
+  const base: StatusFingerprintInput = {
+    status: "healthy",
+    checkStatuses: [check("c1", "healthy"), check("c2", "healthy")],
+  };
+
+  it("is FALSE for a pure timestamp/runs refresh (same vector)", () => {
+    // Volatile fields (evaluatedAt / lastRunAt / runsConsidered) are not part of
+    // the fingerprint, so an object carrying different ones but the same vector
+    // is not a change. Represented here by an identical vector.
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy"), check("c2", "healthy")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(false);
+  });
+
+  it("is TRUE for a per-check flip even when the rollup enum is unchanged", () => {
+    // Rollup stays "healthy" here (contrived), but c2 flipped — the entity view
+    // {status, healthyChecks, totalChecks} could miss this, the fingerprint does not.
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy"), check("c2", "degraded")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(true);
+  });
+
+  it("is TRUE when the check SET changes", () => {
+    const next: StatusFingerprintInput = {
+      status: "healthy",
+      checkStatuses: [check("c1", "healthy")],
+    };
+    expect(statusVectorChanged(base, next)).toBe(true);
+  });
+});

package/src/status-fingerprint.ts

@@ -0,0 +1,66 @@
+/**
+ * Per-check status fingerprint — the change-signal that gates system-health
+ * cache invalidation.
+ *
+ * The cached value is the full aggregated system-health response
+ * (`getSystemHealthStatus`), but most of its fields are VOLATILE: every run
+ * bumps `evaluatedAt`, a check's `lastRunAt`, and `runsConsidered` even when
+ * nothing an operator sees actually changed. Invalidating on those would defeat
+ * the cache (every tick evicts). The fingerprint is invariant to them: it
+ * captures ONLY the derived-status vector a reader gates on:
+ *  - the system-wide rollup `status`, and
+ *  - per check: `configurationId`, its derived `status`, and its slice
+ *    composition (`sliceCount` / `failingSliceCount`).
+ *
+ * So `statusVectorChanged(prev, next)` is true exactly when a check flipped
+ * status, a slice began/stopped failing, or the check set changed — i.e. when a
+ * status a reader renders actually moved. A run that merely refreshes timestamps
+ * with the same vector is NOT a change, so the reconcile skips invalidation and
+ * the cached value (correct except for volatile fields no reader needs) survives
+ * to its TTL. This is the "broaden the change-signal to the per-check status
+ * vector" requirement: it catches a per-check flip that leaves the rollup enum
+ * unchanged (which the entity view's `{status, healthyChecks, totalChecks}`
+ * would miss), while ignoring pure timestamp churn.
+ */
+
+/**
+ * Structural subset of `SystemHealthStatusResponse` the fingerprint reads. The
+ * service's response is assignable to this, so no import (and no cast) is needed
+ * — keeping this module a leaf the cache and executor can both depend on.
+ */
+export interface StatusFingerprintInput {
+  status: string;
+  checkStatuses: readonly {
+    configurationId: string;
+    status: string;
+    sliceCount: number;
+    failingSliceCount: number;
+  }[];
+}
+
+/**
+ * A stable, order-independent fingerprint of a system-health response's derived
+ * status vector. Checks are sorted by `configurationId` so two responses with
+ * the same checks in a different order fingerprint identically.
+ */
+export function statusFingerprint(response: StatusFingerprintInput): string {
+  const perCheck = response.checkStatuses
+    .map(
+      (c) =>
+        `${c.configurationId}:${c.status}:${c.sliceCount}:${c.failingSliceCount}`,
+    )
+    .toSorted();
+  return `${response.status}|${perCheck.join(",")}`;
+}
+
+/**
+ * Whether the derived status vector changed between two responses (ignoring
+ * volatile fields). This is the gate for cache invalidation + cross-pod
+ * broadcast: only a real vector change evicts the cache and wakes other pods.
+ */
+export function statusVectorChanged(
+  previous: StatusFingerprintInput,
+  next: StatusFingerprintInput,
+): boolean {
+  return statusFingerprint(previous) !== statusFingerprint(next);
+}

package/src/status-page/widgets.test.ts

@@ -301,3 +301,87 @@ describe("health widgets — no environment filter (unchanged behavior)", () =>
     expect(result.uptimePct).toBe(50);
   });
 });
+
+/**
+ * Send-time SCOPING regression: the subscriber fan-out only surfaces a HEALTH
+ * notification through a health widget, and only for the systems the widget
+ * CURRENTLY shows (its configured systems ∩ the page's published environments).
+ * Each health widget must therefore declare `subscriptionCategory: "health"` and
+ * a `resolveScopedSystems` that matches what `resolvePublic` renders.
+ */
+describe("health widgets — resolveScopedSystems (send-time scoping)", () => {
+  test("every health widget is tagged with the 'health' subscription category", () => {
+    const widgets = widgetsById();
+    for (const id of ["banner", "systemHealth", "groupStatus", "uptime"]) {
+      expect(widgets.get(id)!.subscriptionCategory).toBe("health");
+    }
+  });
+
+  const scoped = ["prod"];
+
+  test("banner scopes its systems to the published environments", async () => {
+    const widget = widgetsById().get("banner")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const set = await widget.resolveScopedSystems!({
+      config: { systemIds: ["prod-sys", "stage-sys", "both-sys"] },
+      ctx,
+    });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys"]);
+  });
+
+  test("systemHealth scopes to env and applies per-row label overrides in the detailed list", async () => {
+    const widget = widgetsById().get("systemHealth")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const config = {
+      items: [
+        { systemId: "prod-sys", label: "PROD!" },
+        { systemId: "stage-sys" },
+        { systemId: "both-sys" },
+      ],
+    };
+    const set = await widget.resolveScopedSystems!({ config, ctx });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys"]);
+    const detailed = await widget.resolveScopedSystemsDetailed!({ config, ctx });
+    expect(detailed).toEqual([
+      { id: "prod-sys", name: "PROD!" }, // label override wins over catalog name
+      { id: "both-sys", name: "Both System" },
+    ]);
+  });
+
+  test("groupStatus scopes its expanded members to the published environments", async () => {
+    const widget = widgetsById().get("groupStatus")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    const set = await widget.resolveScopedSystems!({
+      config: { groupId: "g1" },
+      ctx,
+    });
+    expect([...set]).toEqual(["prod-sys"]); // g1 = [prod-sys, stage-sys]; prod only
+  });
+
+  test("uptime scopes to its single system, emptying when it is out of scope", async () => {
+    const widget = widgetsById().get("uptime")!;
+    const ctx = makeCtx({ data: DATA, publishedEnvironmentIds: scoped });
+    expect([
+      ...(await widget.resolveScopedSystems!({
+        config: { systemId: "prod-sys", days: 30 },
+        ctx,
+      })),
+    ]).toEqual(["prod-sys"]);
+    expect([
+      ...(await widget.resolveScopedSystems!({
+        config: { systemId: "stage-sys", days: 30 },
+        ctx,
+      })),
+    ]).toEqual([]);
+  });
+
+  test("no environment filter surfaces every configured system", async () => {
+    const widget = widgetsById().get("banner")!;
+    const ctx = makeCtx({ data: DATA }); // all environments
+    const set = await widget.resolveScopedSystems!({
+      config: { systemIds: ["prod-sys", "stage-sys", "both-sys"] },
+      ctx,
+    });
+    expect([...set].toSorted()).toEqual(["both-sys", "prod-sys", "stage-sys"]);
+  });
+});

package/src/status-page/widgets.ts

@@ -68,6 +68,48 @@ async function scopeToEnv(
   return ids.filter((id) => visible.has(id));
 }
 
+/*
+ * The CURRENT set of catalog system ids each health widget surfaces, resolved
+ * from the SAME config the DTO resolve reads and intersected with the page's
+ * published-environment scope. Shared by `resolvePublic` (what the widget shows)
+ * and `resolveScopedSystems` / `resolveScopedSystemsDetailed` (what the
+ * subscriber fan-out may email about), so the shown set and the emailed-about set
+ * can never drift.
+ */
+
+async function bannerScopedIds(
+  config: unknown,
+  ctx: WidgetResolveContext,
+): Promise<string[]> {
+  return scopeToEnv(ctx, BannerConfigSchema.parse(config).systemIds);
+}
+
+async function systemHealthScopedItems(
+  config: unknown,
+  ctx: WidgetResolveContext,
+) {
+  const c = SystemHealthConfigSchema.parse(config);
+  const visible = await envVisibleSystems(ctx);
+  return visible ? c.items.filter((i) => visible.has(i.systemId)) : c.items;
+}
+
+async function groupStatusScopedIds(
+  config: unknown,
+  ctx: WidgetResolveContext,
+): Promise<string[]> {
+  const c = GroupStatusConfigSchema.parse(config);
+  const groups = await allGroups(ctx);
+  const group = groups.find((g) => g.id === c.groupId);
+  return scopeToEnv(ctx, group?.systemIds ?? []);
+}
+
+async function uptimeScopedIds(
+  config: unknown,
+  ctx: WidgetResolveContext,
+): Promise<string[]> {
+  return scopeToEnv(ctx, [UptimeConfigSchema.parse(config).systemId]);
+}
+
 function uptimeToStatus(pct: number): PublicStatus {
   if (pct >= 99.5) return "operational";
   if (pct >= 95) return "degraded";
@@ -228,10 +270,18 @@ const banner: WidgetTypeDefinition = {
   assertBindingsReadable: assertSystems((c) => ({
     systemIds: BannerConfigSchema.parse(c).systemIds,
   })),
+  subscriptionCategory: "health",
+  resolveScopedSystems: async ({ config, ctx }) =>
+    new Set(await bannerScopedIds(config, ctx)),
+  async resolveScopedSystemsDetailed({ config, ctx }) {
+    const ids = await bannerScopedIds(config, ctx);
+    const names = await labelsFor(ctx, ids);
+    return ids.map((id) => ({ id, name: names.get(id) ?? id }));
+  },
   async resolvePublic({ config, ctx }) {
     const c = BannerConfigSchema.parse(config);
     // Omit systems outside the page's published environments before rolling up.
-    const ids = await scopeToEnv(ctx, c.systemIds);
+    const ids = await bannerScopedIds(config, ctx);
     const health = await healthPublicStatuses(ctx, ids);
     const maint = await inMaintenance(ctx, ids);
     const status = overallBannerStatus(
@@ -260,13 +310,26 @@ const systemHealth: WidgetTypeDefinition = {
   assertBindingsReadable: assertSystems((c) => ({
     systemIds: SystemHealthConfigSchema.parse(c).items.map((i) => i.systemId),
   })),
+  subscriptionCategory: "health",
+  async resolveScopedSystems({ config, ctx }) {
+    const items = await systemHealthScopedItems(config, ctx);
+    return new Set(items.map((i) => i.systemId));
+  },
+  async resolveScopedSystemsDetailed({ config, ctx }) {
+    const items = await systemHealthScopedItems(config, ctx);
+    const names = await labelsFor(
+      ctx,
+      items.map((i) => i.systemId),
+    );
+    return items.map((i) => ({
+      id: i.systemId,
+      name: i.label ?? names.get(i.systemId) ?? i.systemId,
+    }));
+  },
   async resolvePublic({ config, ctx }) {
     const c = SystemHealthConfigSchema.parse(config);
     // Drop rows for systems outside the page's published environments.
-    const visible = await envVisibleSystems(ctx);
-    const items = visible
-      ? c.items.filter((i) => visible.has(i.systemId))
-      : c.items;
+    const items = await systemHealthScopedItems(config, ctx);
     const ids = items.map((i) => i.systemId);
     const health = await healthPublicStatuses(ctx, ids);
     const maint = await inMaintenance(ctx, ids);
@@ -330,12 +393,20 @@ const groupStatus: WidgetTypeDefinition = {
   assertBindingsReadable: assertSystems((c) => ({
     groupIds: [GroupStatusConfigSchema.parse(c).groupId],
   })),
+  subscriptionCategory: "health",
+  resolveScopedSystems: async ({ config, ctx }) =>
+    new Set(await groupStatusScopedIds(config, ctx)),
+  async resolveScopedSystemsDetailed({ config, ctx }) {
+    const ids = await groupStatusScopedIds(config, ctx);
+    const names = await labelsFor(ctx, ids);
+    return ids.map((id) => ({ id, name: names.get(id) ?? id }));
+  },
   async resolvePublic({ config, ctx }) {
     const c = GroupStatusConfigSchema.parse(config);
     const groups = await allGroups(ctx);
     const group = groups.find((g) => g.id === c.groupId);
     // Omit group members outside the page's published environments.
-    const ids = await scopeToEnv(ctx, group?.systemIds ?? []);
+    const ids = await groupStatusScopedIds(config, ctx);
     const health = await healthPublicStatuses(ctx, ids);
     const maint = await inMaintenance(ctx, ids);
     const names = await labelsFor(ctx, ids);
@@ -366,6 +437,16 @@ const uptime: WidgetTypeDefinition = {
   assertBindingsReadable: assertSystems((c) => ({
     systemIds: [UptimeConfigSchema.parse(c).systemId],
   })),
+  subscriptionCategory: "health",
+  resolveScopedSystems: async ({ config, ctx }) =>
+    new Set(await uptimeScopedIds(config, ctx)),
+  async resolveScopedSystemsDetailed({ config, ctx }) {
+    const c = UptimeConfigSchema.parse(config);
+    const ids = await uptimeScopedIds(config, ctx);
+    if (ids.length === 0) return [];
+    const names = await labelsFor(ctx, ids);
+    return ids.map((id) => ({ id, name: c.label ?? names.get(id) ?? id }));
+  },
   async resolvePublic({ config, ctx }) {
     const c = UptimeConfigSchema.parse(config);
     const names = await labelsFor(ctx, [c.systemId]);