@checkstack/healthcheck-backend 1.18.0 → 1.20.0

package/src/router-config-secrets.test.ts

@@ -16,7 +16,7 @@ import {
   healthcheckSecretMarker,
   isHealthcheckSecretMarker,
 } from "./config-secrets";
-import type { HealthCheckCache } from "./cache";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 
 /**
  * Guards the SEC-1 fix: `createAndAssign` (the first-check wizard / AI propose
@@ -24,12 +24,7 @@ import type { HealthCheckCache } from "./cache";
  * and return a REDACTED config - never persist or echo a plaintext credential.
  */
 
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+const passthroughCache = createStubHealthCheckCache();
 
 const mockUser = {
   type: "user" as const,

package/src/router-create-and-assign.test.ts

@@ -2,7 +2,7 @@ import { describe, it, expect, mock } from "bun:test";
 import { createHealthCheckRouter } from "./router";
 import { createMockRpcContext } from "@checkstack/backend-api";
 import { call } from "@orpc/server";
-import type { HealthCheckCache } from "./cache";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 
 /**
  * Router-level tests for the atomic create+assign path and the
@@ -12,12 +12,7 @@ import type { HealthCheckCache } from "./cache";
  * `input.body.notificationPolicy`).
  */
 
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+const passthroughCache = createStubHealthCheckCache();
 
 const mockUser = {
   type: "user" as const,

package/src/router-pause-recompute.test.ts

@@ -2,7 +2,7 @@ import { describe, it, expect, mock, beforeEach } from "bun:test";
 import { createHealthCheckRouter } from "./router";
 import { createMockRpcContext } from "@checkstack/backend-api";
 import { call } from "@orpc/server";
-import type { HealthCheckCache } from "./cache";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 
 /**
  * Router-level tests for the pause/resume handlers' rollup-health recompute
@@ -14,12 +14,7 @@ import type { HealthCheckCache } from "./cache";
  * any degraded transition (see the resume handler comment for rationale).
  */
 
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+const passthroughCache = createStubHealthCheckCache();
 
 const mockUser = {
   type: "user" as const,

package/src/router.test.ts

@@ -3,14 +3,9 @@ import { createHealthCheckRouter } from "./router";
 import { createMockRpcContext, Versioned } from "@checkstack/backend-api";
 import { call } from "@orpc/server";
 import { z } from "zod";
-import type { HealthCheckCache } from "./cache";
-
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+import { createStubHealthCheckCache } from "./cache-test-stub";
+
+const passthroughCache = createStubHealthCheckCache();
 
 describe("HealthCheck Router", () => {
   const mockUser = {

package/src/router.ts

@@ -227,6 +227,27 @@ export const createHealthCheckRouter = (opts: {
       });
     }
 
+    // Recompute the rollup `health` entity for this system NOW. Changing an
+    // assignment's environment set can make a previously-effective env slice
+    // orphaned (env disabled/removed from `environmentIds`): that slice stops
+    // producing runs, so NO per-env health-change event will ever fire for it,
+    // and the event-driven rollup consumer would never recompute the disabled
+    // env's last (unhealthy) status away. `getSystemHealthStatus` now excludes
+    // non-effective slices, so this recompute promptly drops the stale slice
+    // from the persisted entity - closing any SLO downtime it was holding open
+    // and clearing the badge - instead of waiting for its runs to age out of
+    // the window. Best-effort: a recompute failure is logged inside the helper
+    // and never breaks the mutation (the live RPC reads are already correct).
+    if (recomputeSystemRollupHealth) {
+      try {
+        await recomputeSystemRollupHealth(args.systemId);
+      } catch (error) {
+        logger.warn(
+          `Failed to recompute rollup health after assignment change for system ${args.systemId}: ${extractErrorMessage(error, "unknown")}`,
+        );
+      }
+    }
+
     // Notify subscribers (e.g., satellite-backend) that assignments changed.
     const emitHook = getEmitHook();
     if (emitHook) {
@@ -473,6 +494,17 @@ export const createHealthCheckRouter = (opts: {
       },
     ),
 
+    getBulkAssignedHealthCheckCounts:
+      os.getBulkAssignedHealthCheckCounts.handler(async ({ input }) => {
+        // ONE grouped query for the whole visible system list (replaces the
+        // per-row getSystemAssociations N+1). recordKey gating on the contract
+        // drops counts for systems the caller may not read.
+        const counts = await service.getBulkAssignedHealthCheckCounts(
+          input.systemIds,
+        );
+        return { counts };
+      }),
+
     associateSystem: os.associateSystem.handler(async ({ input, context }) => {
       await enforceNotGitOpsLocked("System", input.systemId);
       await service.associateSystem({
@@ -573,6 +605,10 @@ export const createHealthCheckRouter = (opts: {
       return service.getRunStats(input);
     }),
 
+    getBulkRunStats: os.getBulkRunStats.handler(async ({ input }) => {
+      return { stats: await service.getBulkRunStats(input) };
+    }),
+
     getDetailedHistory: os.getDetailedHistory.handler(
       async ({ input, context }) => {
         // Handler-side authorization (the contract's `access` is deliberately
@@ -674,9 +710,10 @@ export const createHealthCheckRouter = (opts: {
     ),
     getSystemHealthStatus: os.getSystemHealthStatus.handler(
       async ({ input }) => {
-        const base = await cache.wrapSystemHealthStatus(input.systemId, () =>
-          service.getSystemHealthStatus(input.systemId),
-        );
+        // The cache holds the RAW (pre-override) status; incident overrides are
+        // folded downstream (always live) so an override lifts the instant its
+        // incident resolves. See ./cache.ts for the key/TTL/invalidation contract.
+        const base = await cache.read(input.systemId);
         const folded = await foldIncidentOverrides({
           [input.systemId]: base,
         });
@@ -688,24 +725,15 @@ export const createHealthCheckRouter = (opts: {
       async ({ input }) => {
         // Per-entity caching: each system's status is cached individually
         // and invalidated by id on mutations, so dashboards with overlapping
-        // (but non-identical) system sets share cache entries. See
-        // ./cache.ts for the key/TTL/invalidation contract.
-        const statuses: Record<string, SystemHealthStatusResponse> = {};
-        await Promise.all(
-          input.systemIds.map(async (systemId) => {
-            statuses[systemId] = await cache.wrapSystemHealthStatus(
-              systemId,
-              () => service.getSystemHealthStatus(systemId),
-            );
-          }),
-        );
+        // (but non-identical) system sets share cache entries.
+        const statuses = await cache.readBulk(input.systemIds);
         return { statuses: await foldIncidentOverrides(statuses) };
       },
     ),
 
     getBulkSystemHealthMatrix: os.getBulkSystemHealthMatrix.handler(
       async ({ input }) => {
-        const matrix = await service.getBulkSystemHealthMatrix(input.systemIds);
+        const matrix = await cache.readMatrix(input.systemIds);
 
         // Fold active incident overrides into each system's OVERALL rollup, so
         // an incident-forced status still propagates through dependencies (as

package/src/schema.ts

@@ -127,6 +127,13 @@ export const systemHealthChecks = pgTable(
   },
   (t) => ({
     pk: primaryKey({ columns: [t.systemId, t.configurationId] }),
+    // Reverse lookup for getSystemIdsForConfiguration on config-change
+    // recompute (WHERE configuration_id [AND enabled]). The PK leads with
+    // system_id, so a config-scoped scan cannot use it.
+    configEnabledIdx: index("system_health_checks_config_enabled_idx").on(
+      t.configurationId,
+      t.enabled,
+    ),
   }),
 );
 
@@ -186,37 +193,65 @@ export const healthCheckStateTransitions = pgTable(
   }),
 );
 
-export const healthCheckRuns = pgTable("health_check_runs", {
-  id: uuid("id").primaryKey().defaultRandom(),
-  configurationId: uuid("configuration_id")
-    .notNull()
-    .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
-  systemId: text("system_id").notNull(),
-  /**
-   * Environment this run was executed for (per-environment fan-out).
-   * null = ran with no environment (the opt-out / no-membership case,
-   * which is exactly the pre-feature behavior). Nullable text, NOT a FK
-   * to the catalog `environments` table (healthcheck and catalog are
-   * separate plugins with separate Postgres schemas, mirroring how
-   * `systemId` is a bare text with no FK to `systems`).
-   */
-  environmentId: text("environment_id"),
-  status: healthCheckStatusEnum("status").notNull(),
-  /** Execution duration in milliseconds */
-  latencyMs: integer("latency_ms"),
-  result: jsonb("result").$type<Record<string, unknown>>(),
-  /**
-   * Source identifier for result attribution.
-   * null = local core execution, UUID = satellite ID.
-   */
-  sourceId: text("source_id"),
-  /**
-   * Human-readable source label for UI display.
-   * e.g. "Local" or "EU West (eu-west-1)".
-   */
-  sourceLabel: text("source_label"),
-  timestamp: timestamp("timestamp").defaultNow().notNull(),
-});
+export const healthCheckRuns = pgTable(
+  "health_check_runs",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    configurationId: uuid("configuration_id")
+      .notNull()
+      .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
+    systemId: text("system_id").notNull(),
+    /**
+     * Environment this run was executed for (per-environment fan-out).
+     * null = ran with no environment (the opt-out / no-membership case,
+     * which is exactly the pre-feature behavior). Nullable text, NOT a FK
+     * to the catalog `environments` table (healthcheck and catalog are
+     * separate plugins with separate Postgres schemas, mirroring how
+     * `systemId` is a bare text with no FK to `systems`).
+     */
+    environmentId: text("environment_id"),
+    status: healthCheckStatusEnum("status").notNull(),
+    /** Execution duration in milliseconds */
+    latencyMs: integer("latency_ms"),
+    result: jsonb("result").$type<Record<string, unknown>>(),
+    /**
+     * Source identifier for result attribution.
+     * null = local core execution, UUID = satellite ID.
+     */
+    sourceId: text("source_id"),
+    /**
+     * Human-readable source label for UI display.
+     * e.g. "Local" or "EU West (eu-west-1)".
+     */
+    sourceLabel: text("source_label"),
+    timestamp: timestamp("timestamp").defaultNow().notNull(),
+  },
+  (t) => ({
+    // The status read path reads the last N runs for a (system, check) slice
+    // ordered by timestamp DESC. Without a supporting index every such read
+    // sequential-scans the whole (multi-million-row) table and sorts. Postgres
+    // scans a plain btree backward, so an ASC index serves ORDER BY ... DESC.
+    //
+    // Cross-environment newest-run read (WHERE system_id, configuration_id
+    // ORDER BY timestamp DESC LIMIT n - the single hottest query) and the
+    // retention DELETE (WHERE system_id, configuration_id AND timestamp < cutoff).
+    checkRecentIdx: index("health_check_runs_check_recent_idx").on(
+      t.systemId,
+      t.configurationId,
+      t.timestamp,
+    ),
+    // Env-scoped slice read (WHERE system_id, configuration_id,
+    // environment_id [= ? | IS NULL] ORDER BY timestamp DESC LIMIT n), the
+    // per-check DISTINCT-environment discovery, and the per-env "last healthy"
+    // max(timestamp) GROUP BY environment_id.
+    sliceRecentIdx: index("health_check_runs_slice_recent_idx").on(
+      t.systemId,
+      t.configurationId,
+      t.environmentId,
+      t.timestamp,
+    ),
+  }),
+);
 
 /**
  * Bucket size enum for aggregated data.
@@ -286,5 +321,13 @@ export const healthCheckAggregates = pgTable(
       t.bucketSize,
       t.sourceId,
     ).nullsNotDistinct(),
+    // Health-state read with configuration_id absent (WHERE system_id,
+    // bucket_size, bucket_start ...). The unique index leads with
+    // configuration_id, so this query scans without a system-leading index.
+    systemBucketIdx: index("health_check_aggregates_system_bucket_idx").on(
+      t.systemId,
+      t.bucketSize,
+      t.bucketStart,
+    ),
   }),
 );

package/src/service-batching.test.ts

@@ -18,6 +18,8 @@ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
       enabled: true,
       paused: false,
       stateThresholds: null,
+      // All-environments selector; each check has only the env-less slice below.
+      environmentIds: null,
     }));
     const assocWhere = mock(() => Promise.resolve(associations));
     const assocInnerJoin = Object.assign(Promise.resolve([]), {
@@ -39,6 +41,11 @@ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
       orderBy: runsOrderBy,
     });
 
+    // Distinct env keys query per check: a single env-less (null) slice.
+    const distinctFrom = Object.assign(Promise.resolve([]), {
+      where: mock(() => Promise.resolve([{ environmentId: null }])),
+    });
+
     let selectCallCount = 0;
     const db = withTransactionMock({
       select: mock(() => {
@@ -46,6 +53,7 @@ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
         if (selectCallCount === 1) return { from: mock(() => assocFrom) };
         return { from: mock(() => runsFrom) };
       }),
+      selectDistinct: mock(() => ({ from: mock(() => distinctFrom) })),
       insert: mock(() => ({ values: mock(() => Promise.resolve()) })),
       update: mock(() => ({
         set: mock(() => ({ where: mock(() => Promise.resolve()) })),

package/src/service-bulk-counts.it.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Integration test for `HealthCheckService.getBulkAssignedHealthCheckCounts`
+ * against a REAL Postgres. The method's whole point is the GROUP BY / COUNT and
+ * the zero-fill for systems with no rows - behaviour a mocked db cannot prove
+ * (the database does the grouping). These tests pin: counts are grouped per
+ * system, systems with no assignments report 0, requested-but-absent systems
+ * report 0, and non-requested systems never leak in.
+ *
+ * Gated on CHECKSTACK_IT so it runs in CI (shared compose Postgres) and is
+ * skipped in the default `bun test` run, matching the other *.it.test.ts here.
+ */
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { Pool } from "pg";
+import type {
+  SafeDatabase,
+  HealthCheckRegistry,
+  CollectorRegistry,
+} from "@checkstack/backend-api";
+import * as schema from "./schema";
+import { HealthCheckService } from "./service";
+
+const PG_URL =
+  process.env.CHECKSTACK_IT_PG_URL ??
+  "postgres://postgres:postgres@localhost:5432/postgres";
+const SCHEMA = "healthcheck_it_bulk_counts";
+
+let admin: Pool;
+let pool: Pool;
+let service: HealthCheckService;
+
+async function insertAssignment(row: {
+  systemId: string;
+  configurationId: string;
+}): Promise<void> {
+  await pool.query(
+    `INSERT INTO "${SCHEMA}".system_health_checks (system_id, configuration_id)
+     VALUES ($1, $2)`,
+    [row.systemId, row.configurationId],
+  );
+}
+
+describe.skipIf(!process.env.CHECKSTACK_IT)(
+  "HealthCheckService.getBulkAssignedHealthCheckCounts (shared Postgres)",
+  () => {
+    beforeAll(async () => {
+      admin = new Pool({ connectionString: PG_URL });
+      await admin.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
+      await admin.query(`CREATE SCHEMA "${SCHEMA}"`);
+      // Only the columns the grouped COUNT touches; no FK to configurations so
+      // the DDL stays minimal and focused on the aggregation behaviour.
+      await admin.query(
+        `CREATE TABLE "${SCHEMA}".system_health_checks (
+          system_id text NOT NULL,
+          configuration_id uuid NOT NULL,
+          PRIMARY KEY (system_id, configuration_id)
+        )`,
+      );
+      pool = new Pool({
+        connectionString: PG_URL,
+        options: `-c search_path=${SCHEMA}`,
+      });
+      const db = drizzle(pool, {
+        schema,
+      }) as unknown as SafeDatabase<typeof schema>;
+      // registry / collectorRegistry are unused by the count method under test;
+      // stub them so the constructor is satisfied without wiring real registries.
+      const service_ = new HealthCheckService(
+        db,
+        {} as unknown as HealthCheckRegistry,
+        {} as unknown as CollectorRegistry,
+      );
+      service = service_;
+    });
+
+    afterAll(async () => {
+      await pool?.end();
+      await admin.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
+      await admin.end();
+    });
+
+    beforeEach(async () => {
+      await pool.query(`TRUNCATE "${SCHEMA}".system_health_checks`);
+    });
+
+    it("groups counts per system and zero-fills systems with no assignments", async () => {
+      // sys-a: 2 assignments, sys-b: 1, sys-c: none.
+      await insertAssignment({
+        systemId: "sys-a",
+        configurationId: crypto.randomUUID(),
+      });
+      await insertAssignment({
+        systemId: "sys-a",
+        configurationId: crypto.randomUUID(),
+      });
+      await insertAssignment({
+        systemId: "sys-b",
+        configurationId: crypto.randomUUID(),
+      });
+
+      const counts = await service.getBulkAssignedHealthCheckCounts([
+        "sys-a",
+        "sys-b",
+        "sys-c",
+      ]);
+
+      expect(counts).toEqual({ "sys-a": 2, "sys-b": 1, "sys-c": 0 });
+    });
+
+    it("returns 0 for a requested system that has no rows at all", async () => {
+      await insertAssignment({
+        systemId: "sys-a",
+        configurationId: crypto.randomUUID(),
+      });
+
+      const counts = await service.getBulkAssignedHealthCheckCounts([
+        "sys-a",
+        "missing",
+      ]);
+
+      expect(counts).toEqual({ "sys-a": 1, missing: 0 });
+    });
+
+    it("never leaks counts for systems not in the requested set", async () => {
+      await insertAssignment({
+        systemId: "sys-a",
+        configurationId: crypto.randomUUID(),
+      });
+      await insertAssignment({
+        systemId: "sys-other",
+        configurationId: crypto.randomUUID(),
+      });
+
+      const counts = await service.getBulkAssignedHealthCheckCounts(["sys-a"]);
+
+      expect(counts).toEqual({ "sys-a": 1 });
+      expect(Object.keys(counts)).not.toContain("sys-other");
+    });
+
+    it("returns an empty map for an empty request without querying", async () => {
+      expect(await service.getBulkAssignedHealthCheckCounts([])).toEqual({});
+    });
+  },
+);