@checkstack/healthcheck-backend 1.18.0 → 1.20.0

package/src/index.ts

@@ -230,6 +230,10 @@ export default createBackendPlugin({
           throw new Error("Catalog client not initialized");
         return gitopsCatalogClient;
       },
+      // Lazy: the cache is created in init() (after this register() call) and
+      // reconcile only runs post-init, so `healthCheckCache` is populated by the
+      // time GitOps mutations fire.
+      getCache: () => healthCheckCache,
     });
 
     env.registerInit({
@@ -461,18 +465,34 @@ export default createBackendPlugin({
           execute: deferredProjectionExecute,
         });
 
+        // Per-entity status cache shared between the router, queue executor,
+        // AI signals contributor, and afterPluginsReady cleanup hooks. It is the
+        // single sanctioned reader (compute-on-read via `service`) AND
+        // invalidator of a system's derived health status. It runs on the
+        // platform CacheManager, so cross-pod coherence comes from the SHARED
+        // backend (a distributed provider such as Redis makes an eviction
+        // visible to every pod); no application-level broadcast is needed.
+        const cache = createHealthCheckCache({
+          cacheManager,
+          logger,
+          service,
+        });
+        healthCheckCache = cache;
+
         // Contribute this plugin's per-system health problems to the AI
         // `system.issues` aggregator. PER-SOURCE access is OUR job: gate on the
         // principal's `healthcheck.status` grant and return {} (never throw)
-        // when not satisfied. The read derives from the durable
-        // `health_check_runs` / `system_health_checks` tables (global, identical
-        // on every pod) and reuses the SAME pure deriver as the dashboard
-        // filler, so backend signals match the UI's source/tone/label/detail.
+        // when not satisfied. The candidate set derives from the durable
+        // `system_health_checks` table (global, identical on every pod); the
+        // per-system status reads go through the SHARED CACHE (reusing the warm
+        // badge/dashboard entries) and reuse the SAME pure deriver as the
+        // dashboard filler, so backend signals match the UI.
         env
           .getExtensionPoint(systemSignalsExtensionPoint)
           .contribute(
             createHealthcheckSignalsContributor({
-              service,
+              candidateSource: service,
+              cache,
               resolver: createSystemAccessResolver(rpcClient),
             }),
           );
@@ -492,13 +512,6 @@ export default createBackendPlugin({
         // Create gitops client for provenance lock checks
         const gitOpsClient = rpcClient.forPlugin(GitOpsApi);
 
-        // Per-entity status cache shared between the router, queue executor,
-        // and afterPluginsReady cleanup hooks. Mutations / new check results
-        // invalidate by systemId BEFORE emitting signals so frontend
-        // refetches see fresh data.
-        const cache = createHealthCheckCache({ cacheManager, logger });
-        healthCheckCache = cache;
-
         // Setup queue-based health check worker
         await setupHealthCheckWorker({
           notificationClient,
@@ -630,6 +643,12 @@ export default createBackendPlugin({
       }) => {
         // Store emitHook for the queue worker (Closure-based Hook Getter pattern)
         storedEmitHook = emitHook;
+
+        // No cross-pod status-cache broadcast: the status cache runs on the
+        // platform CacheManager, so a distributed backend (Redis) makes every
+        // eviction visible to all pods through the shared store. The prior
+        // per-pod-cache + broadcast layer was removed in favour of that.
+
         // Converge the per-environment recurring job set at boot (schedule
         // desired (config, system, env) jobs, cancel orphans incl. old-format
         // ones). The periodic reconcile below keeps it converged as catalog

package/src/queue-executor.test.ts

@@ -5,16 +5,11 @@ import {
   recomputeSystemRollupHealth,
   type HealthCheckJobPayload,
 } from "./queue-executor";
-import type { HealthCheckCache } from "./cache";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 import { SuspectLane } from "./suspect-lane";
 import type { SlowCheckRuntime } from "./slow-check-config";
 
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+const passthroughCache = createStubHealthCheckCache();
 
 // Pass-through advisory lock: these tests don't exercise cross-pod
 // serialization, so run the critical section directly.
@@ -248,16 +243,7 @@ describe("Queue-Based Health Check Executor", () => {
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
         if (selectCallCount === 1) {
-          // First call: get previous system health status
-          return {
-            from: mock(() => ({
-              innerJoin: mock(() => ({
-                where: mock(() => Promise.resolve([])),
-              })),
-            })),
-          };
-        } else if (selectCallCount === 2) {
-          // Second call: fetch configuration (return paused config)
+          // First call: fetch configuration (return paused config)
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -371,7 +357,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -592,7 +578,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -762,7 +748,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock((...args: unknown[]) => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -1129,7 +1115,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock((...args: unknown[]) => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -1323,7 +1309,7 @@ describe("executeHealthCheckJob - structured assertion outcomes", () => {
     let selectCallCount = 0;
     (mockDb.select as any) = mock(() => {
       selectCallCount++;
-      if (selectCallCount === 2) {
+      if (selectCallCount === 1) {
         return {
           from: mock(() => ({
             innerJoin: mock(() => ({
@@ -1534,13 +1520,14 @@ describe("executeHealthCheckJob - slow-check bulkhead wiring", () => {
     (mockCatalogClient.getSystem as any) = mock(async () => ({ id: "system-1", name: "web-01" }));
     (mockCatalogClient as any).resolveSystemEnvironments = mock(async () => []);
 
-    // Select call order: #1 getSystemHealthStatus (rollup prev), #2 config,
-    // #3 fetchRecentRunsForSlice. Everything else falls through to default.
+    // Select call order: #1 config, #2 fetchRecentRunsForSlice. Everything else
+    // falls through to default. (The eager rollup-prev read that used to be #1
+    // is now computed lazily only on the catastrophic-failure path.)
     const defaultSelect = mockDb.select;
     let selectCallCount = 0;
     (mockDb.select as any) = mock((...args: unknown[]) => {
       selectCallCount++;
-      if (selectCallCount === 2) {
+      if (selectCallCount === 1) {
         return {
           from: mock(() => ({
             innerJoin: mock(() => ({
@@ -1567,7 +1554,7 @@ describe("executeHealthCheckJob - slow-check bulkhead wiring", () => {
           })),
         };
       }
-      if (selectCallCount === 3) {
+      if (selectCallCount === 2) {
         return {
           from: mock(() => ({
             where: mock(() => ({

package/src/queue-executor.ts

@@ -35,20 +35,17 @@ import {
 } from "@checkstack/healthcheck-common";
 import {
   CatalogApi,
-  catalogRoutes,
-  createSystemSubject,
   type Environment,
 } from "@checkstack/catalog-common";
 import {
   resolveEffectiveEnvironments,
   type EffectiveEnvironment,
 } from "./effective-environments";
-import { systemHealthCollapseKey } from "@checkstack/healthcheck-common";
+import { buildHealthTransitionNotification } from "./health-notification-content";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import { IncidentApi } from "@checkstack/incident-common";
 import { NotificationApi } from "@checkstack/notification-common";
-import { healthcheckSystemSubscription } from "@checkstack/healthcheck-common";
-import { resolveRoute, type InferClient, extractErrorMessage} from "@checkstack/common";
+import { type InferClient, extractErrorMessage} from "@checkstack/common";
 import { secretEnvMappingSchema } from "@checkstack/secrets-common";
 import type {
   SecretResolverService,
@@ -382,18 +379,20 @@ export async function recomputeSystemRollupHealth(args: {
   // pure entity-recompute path (pause/resume) skips the extra prev read.
   const wantsSignal = signalService !== undefined || cache !== undefined;
   try {
-    let previousStatus: HealthCheckStatus | undefined;
+    // Capture the FULL rollup states (not just the status enum) so the cache
+    // reconcile can gate on the per-check vector while the frontend signal
+    // stays gated on the coarser rollup-enum transition.
+    let previousState: AggregatedHealth | undefined;
     if (wantsSignal) {
-      const previousState = await service.getSystemHealthStatus(systemId);
-      previousStatus = previousState.status;
+      previousState = await service.getSystemHealthStatus(systemId);
     }
-    let newStatus: HealthCheckStatus | undefined = previousStatus;
+    let newState: AggregatedHealth | undefined = previousState;
     await writeHealthEntity({
       handle: getHealthEntity?.(),
       entityId: rollupEntityId,
       apply: async () => {
         const rollupState = await service.getSystemHealthStatus(systemId);
-        newStatus = rollupState.status;
+        newState = rollupState;
         return toHealthEntityView(rollupState);
       },
       serialize: makeHealthSerializer(rollupEntityId),
@@ -404,21 +403,23 @@ export async function recomputeSystemRollupHealth(args: {
         ),
     });
 
-    if (
-      wantsSignal &&
-      previousStatus !== undefined &&
-      newStatus !== undefined &&
-      newStatus !== previousStatus
-    ) {
-      await cache?.invalidateSystem(systemId);
-      await signalService?.broadcast(SYSTEM_STATUS_CHANGED, {
-        systemId,
-        previousStatus,
-        newStatus,
-      });
+    if (wantsSignal && previousState !== undefined && newState !== undefined) {
+      // Cache: evict the rollup key + broadcast to the cluster on ANY per-check
+      // vector change — a check that flips while the rollup enum stays put still
+      // changes the rollup's `checkStatuses`, and a reader gets that vector.
+      await cache?.reconcile({ systemId, previous: previousState, next: newState });
+      // Frontend signal: only a rollup-enum transition moves the badge, so a
+      // per-check-only change needs no SYSTEM_STATUS_CHANGED refetch signal.
+      if (newState.status !== previousState.status) {
+        await signalService?.broadcast(SYSTEM_STATUS_CHANGED, {
+          systemId,
+          previousStatus: previousState.status,
+          newStatus: newState.status,
+        });
+      }
     }
-    return previousStatus !== undefined && newStatus !== undefined
-      ? { previousStatus, newStatus }
+    return previousState !== undefined && newState !== undefined
+      ? { previousStatus: previousState.status, newStatus: newState.status }
       : undefined;
   } catch (error) {
     // A recompute failure must never break the pause/resume RPC. The
@@ -465,6 +466,13 @@ async function notifyStateChange(props: {
   systemId: string;
   systemName: string;
   configurationId: string;
+  /**
+   * Human-readable name of the health check whose run drove this transition.
+   * Named in the body and surfaced as a `healthcheck.healthcheck` subject so
+   * subscribers see WHICH check failed, not just which system. Best-effort:
+   * falls back to the `configurationId` when the name could not be resolved.
+   */
+  configurationName?: string;
   previousStatus: HealthCheckStatus;
   newStatus: HealthCheckStatus;
   /**
@@ -493,6 +501,7 @@ async function notifyStateChange(props: {
     systemId,
     systemName,
     configurationId,
+    configurationName,
     previousStatus,
     newStatus,
     environmentId,
@@ -505,8 +514,9 @@ async function notifyStateChange(props: {
     logger,
   } = props;
 
-  const envScoped = typeof environmentId === "string";
-  const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
+  // The check that just ran is the one driving this aggregate transition, so
+  // its name is the authoritative check to blame. Fall back to the id.
+  const checkName = configurationName ?? configurationId;
 
   const transition = classifyTransition(previousStatus, newStatus);
   if (transition === "none") {
@@ -572,70 +582,24 @@ async function notifyStateChange(props: {
     );
   }
 
-  let title: string;
-  let body: string;
-  let importance: "info" | "warning" | "critical";
-
-  if (transition === "recovery") {
-    title = `System health restored${envSuffix}: ${systemName}`;
-    body = envScoped
-      ? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
-      : `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
-    importance = "info";
-  } else if (newStatus === "unhealthy") {
-    title = `System health critical${envSuffix}: ${systemName}`;
-    body = envScoped
-      ? `Health checks indicate **${systemName}** is unhealthy in environment **${environmentName ?? environmentId}** and may be down in that environment.`
-      : `Health checks indicate **${systemName}** is unhealthy and may be down.`;
-    importance = "critical";
-  } else {
-    // degraded — either an escalation from healthy or a partial recovery
-    title = `System health degraded${envSuffix}: ${systemName}`;
-    body = envScoped
-      ? `Some health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are failing. That environment may be experiencing issues.`
-      : `Some health checks for **${systemName}** are failing. The system may be experiencing issues.`;
-    importance = "warning";
-  }
-
-  const systemDetailPath = resolveRoute(catalogRoutes.routes.systemDetail, {
-    systemId,
-  });
-  // Recovery lands on the default (all) view; failing transitions deep-link
-  // operators into the failing-checks filter so they can debug immediately.
-  const actionUrl =
-    transition === "recovery"
-      ? systemDetailPath
-      : `${systemDetailPath}?filter=failing`;
-  const actionLabel =
-    transition === "recovery" ? "View System" : "View failing checks";
-
   void catalogClient; // parents are resolved server-side via stored target edges
 
   try {
-    await notificationClient.notifyForSubscription({
-      specId: healthcheckSystemSubscription.specId,
-      resourceKeys: [systemId],
-      title,
-      body,
-      importance,
-      action: { label: actionLabel, url: actionUrl },
-      // Env-qualified collapse key so two failing envs of one system generate
-      // two independent notification cards (one per env) instead of merging
-      // -> operators see all env outages. The system-rollup transition
-      // (`environmentId === null`/undefined) keys on the bare systemId and
-      // therefore reuses the pre-existing single-card identity.
-      collapseKey: envScoped
-        ? systemHealthCollapseKey(systemId, environmentId)
-        : systemHealthCollapseKey(systemId),
-      subjects: [
-        createSystemSubject({
-          id: systemId,
-          name: systemName,
-          url: systemDetailPath,
-          status: newStatus,
-        }),
-      ],
-    });
+    // Content (title/body/subjects/collapseKey) is built by a pure, unit-tested
+    // helper so the wording - which now NAMES the failing check and pushes a
+    // `healthcheck.healthcheck` subject - can be verified without the executor.
+    await notificationClient.notifyForSubscription(
+      buildHealthTransitionNotification({
+        transition,
+        systemId,
+        systemName,
+        configurationId,
+        checkName,
+        newStatus,
+        environmentId,
+        environmentName,
+      }),
+    );
     logger.debug(
       `Notified subscribers: ${previousStatus} → ${newStatus} for system ${systemId}`,
     );
@@ -732,13 +696,14 @@ async function executeHealthCheckJob(props: {
   // other.
   const makeHealthSerializer = createHealthEntitySerializer({ advisoryLock });
 
-  // The system-rollup status BEFORE this tick (all environments + env-less).
-  // Captured once so the post-loop rollup write (§7.4.3) — and the
-  // catastrophic-failure path — can record a correct prev → next rollup
-  // transition (environmentId = null). This is the system-wide aggregate read
-  // the executor has always taken first.
-  const rollupPreviousState = await service.getSystemHealthStatus(systemId);
-  const rollupPreviousStatus = rollupPreviousState.status;
+  // NOTE: the system-rollup status BEFORE this tick is computed LAZILY, only on
+  // the catastrophic-failure path that actually consumes it (see the `catch`
+  // below). It used to be captured here on EVERY run - a full worst-wins rollup
+  // (`getSystemHealthStatus(systemId)`, an N+1 across every check × environment)
+  // - even though the normal success/failure paths record their transition from
+  // the per-env pre-read (`previousState`, below) and never touch the rollup
+  // pre-state. Deferring it to the rare error path removes that whole recompute
+  // from the hot path of every check tick.
 
   // Slow-check lane admission (set when this run was admitted to the suspect
   // lane); released in the outer finally so the slot frees on any exit path.
@@ -994,14 +959,16 @@ async function executeHealthCheckJob(props: {
       const envEntityId = encodeHealthEntityId({ systemId, environmentId });
       const serializeEnvWrite = makeHealthSerializer(envEntityId);
 
-      // Per-env baseline status for the transition log: the env-scoped
-      // aggregate BEFORE this run. Computed per env so a transition row is
-      // recorded against the right (system, environment) streak.
-      const previousState = await service.getSystemHealthStatus(
-        systemId,
-        environmentId,
-      );
-      const previousStatus = previousState.status;
+      // Per-env baseline: the env-scoped aggregate BEFORE this run. Read INSIDE
+      // the serialized `apply` below (assigned to these vars), NOT here — so a
+      // concurrent same-slice run cannot commit between the baseline read and
+      // our own insert. If it were read here (outside the `health:<envEntityId>`
+      // lock), the cache change-gate could compare `next` against a baseline a
+      // sibling run already superseded and miss a real transition, stranding a
+      // stale cached status until the TTL. Assigned by whichever branch's
+      // `apply` runs; used for the transition log AND the cache reconcile.
+      let previousState!: AggregatedHealth;
+      let previousStatus!: HealthCheckStatus;
 
       // Curated, read-only run-context metadata exposed to collectors.
       // Metadata only - never secrets or config. `environment` carries the
@@ -1309,6 +1276,13 @@ async function executeHealthCheckJob(props: {
         handle: getHealthEntity?.(),
         entityId: envEntityId,
         apply: async () => {
+          // In-lock pre-run baseline (see the `previousState` declaration): read
+          // here, inside the serialized critical section, before the insert.
+          previousState = await service.getSystemHealthStatus(
+            systemId,
+            environmentId,
+          );
+          previousStatus = previousState.status;
           // §perf: batch the run INSERT + aggregate SELECT/UPSERT under ONE
           // `SET LOCAL search_path` transaction (3 scoped-db transactions → 1),
           // which also makes the run and its aggregate commit atomically.
@@ -1356,9 +1330,18 @@ async function executeHealthCheckJob(props: {
         `Health check ${configId} for system ${systemId} failed: ${finalError}`,
       );
 
-      // Invalidate the per-system status cache before broadcasting so any
-      // frontend that refetches in response to the signal gets fresh data.
-      await cache.invalidateSystem(systemId);
+      // Reconcile this environment's cached status: evict + broadcast to the
+      // cluster ONLY when the per-check vector actually changed (a run that
+      // leaves every check's status unchanged keeps the cache warm instead of
+      // thrashing it every tick). The rollup key is reconciled separately by
+      // the debounced rollup consumer (recomputeSystemRollupHealth), also
+      // vector-gated.
+      await cache.reconcile({
+        systemId,
+        environmentId,
+        previous: previousState,
+        next: newState,
+      });
 
       await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
         systemId,
@@ -1390,6 +1373,7 @@ async function executeHealthCheckJob(props: {
           systemId,
           systemName,
           configurationId: configId,
+          configurationName: configRow.configName,
           previousStatus,
           newStatus: newState.status,
           environmentId,
@@ -1466,6 +1450,13 @@ async function executeHealthCheckJob(props: {
       handle: getHealthEntity?.(),
       entityId: envEntityId,
       apply: async () => {
+        // In-lock pre-run baseline (see the `previousState` declaration): read
+        // here, inside the serialized critical section, before the insert.
+        previousState = await service.getSystemHealthStatus(
+          systemId,
+          environmentId,
+        );
+        previousStatus = previousState.status;
         // §perf: batch the run INSERT + aggregate SELECT/UPSERT under ONE
         // `SET LOCAL search_path` transaction (3 scoped-db transactions → 1),
         // which also makes the run and its aggregate commit atomically.
@@ -1512,9 +1503,16 @@ async function executeHealthCheckJob(props: {
       `Ran health check ${configId} for system ${systemId}: ${result.status}`,
     );
 
-    // Invalidate the per-system status cache before broadcasting so any
-    // frontend that refetches in response to the signal gets fresh data.
-    await cache.invalidateSystem(systemId);
+    // Reconcile this environment's cached status: evict + broadcast to the
+    // cluster ONLY when the per-check vector actually changed (a steady-state
+    // healthy run keeps the cache warm). The rollup key is reconciled by the
+    // debounced rollup consumer (recomputeSystemRollupHealth), also vector-gated.
+    await cache.reconcile({
+      systemId,
+      environmentId,
+      previous: previousState,
+      next: newState,
+    });
 
     // Broadcast enriched signal for realtime frontend updates (e.g., terminal feed)
     await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
@@ -1557,6 +1555,7 @@ async function executeHealthCheckJob(props: {
         systemId,
         systemName,
         configurationId: configId,
+        configurationName: configRow.configName,
         previousStatus,
         newStatus: newState.status,
         environmentId,
@@ -1620,12 +1619,19 @@ async function executeHealthCheckJob(props: {
     // the system-level health change still emits. Reuses the pre-tick
     // rollup status captured before the try block.
     const rollupEntityId = encodeHealthEntityId({ systemId });
-    const previousStatus = rollupPreviousStatus;
+    // The pre-failure rollup baseline. Read INSIDE `apply` (inside the
+    // `health:<systemId>` lock), before the failure-run insert, so a concurrent
+    // catastrophic tick for the same system can't commit between the baseline
+    // read and this insert and make the cache change-gate miss a transition.
+    let rollupPreState!: AggregatedHealth;
+    let previousStatus!: HealthCheckStatus;
     let newState!: AggregatedHealth;
     await writeHealthEntity({
       handle: getHealthEntity?.(),
       entityId: rollupEntityId,
       apply: async () => {
+        rollupPreState = await service.getSystemHealthStatus(systemId);
+        previousStatus = rollupPreState.status;
         // §perf: batch the failure run INSERT + aggregate SELECT/UPSERT under
         // ONE `SET LOCAL search_path` transaction (3 scoped-db transactions →
         // 1), which also makes them commit atomically.
@@ -1684,9 +1690,15 @@ async function executeHealthCheckJob(props: {
       // Use IDs as fallback
     }
 
-    // Invalidate the per-system status cache before broadcasting so any
-    // frontend that refetches in response to the signal gets fresh data.
-    await cache.invalidateSystem(systemId);
+    // Reconcile the rollup cache: evict + broadcast only on a real vector
+    // change. This catastrophic path writes the bare `<systemId>` entity (it IS
+    // the rollup), so it owns the rollup key directly — no debounced consumer
+    // runs for it.
+    await cache.reconcile({
+      systemId,
+      previous: rollupPreState,
+      next: newState,
+    });
 
     // Broadcast enriched failure signal for realtime frontend updates
     await signalService.broadcast(HEALTH_CHECK_RUN_COMPLETED, {
@@ -1723,6 +1735,7 @@ async function executeHealthCheckJob(props: {
         systemId,
         systemName,
         configurationId: configId,
+        configurationName: configName,
         previousStatus,
         newStatus: newState.status,
         service,

package/src/retention-job.ts

@@ -153,6 +153,14 @@ async function deleteExpiredRawRuns(params: DeleteExpiredRawRunsParams) {
   const cutoffDate = new Date();
   cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
 
+  // Status-cache note: this delete does NOT invalidate the system-health status
+  // cache, and deliberately need not. `evaluateHealthStatus` derives status from
+  // the most-recent-N runs (count-based, no time component), and the cutoff is
+  // days old, so for any ACTIVELY-running check the deleted rows are already
+  // outside the evaluation window — removing them cannot change the current
+  // derived status. The only case it could is a check that STOPPED running long
+  // enough for its entire history to age past the cutoff; that flip is bounded by
+  // the 15s status-cache TTL (an acceptable, rare edge for a stopped check).
   await db
     .delete(healthCheckRuns)
     .where(

package/src/rollup-consumer.test.ts

@@ -35,7 +35,7 @@ interface Harness {
   changeHandler: (change: EntityChanged) => Promise<void>;
   getSystemHealthStatus: ReturnType<typeof mock>;
   broadcast: ReturnType<typeof mock>;
-  invalidateSystem: ReturnType<typeof mock>;
+  reconcile: ReturnType<typeof mock>;
 }
 
 async function setup(opts: {
@@ -75,7 +75,7 @@ async function setup(opts: {
   }) as unknown as OnEntityChanged;
 
   const broadcast = mock(async () => {});
-  const invalidateSystem = mock(async () => {});
+  const reconcile = mock(async () => {});
 
   await setupRollupConsumer({
     queueManager,
@@ -85,7 +85,7 @@ async function setup(opts: {
       withXactLock: async ({ fn }: { fn: () => Promise<unknown> }) => fn(),
     } as never,
     signalService: { broadcast } as never,
-    cache: { invalidateSystem } as never,
+    cache: { reconcile } as never,
     // No entity handle: writeHealthEntity runs `apply` directly.
     getHealthEntity: () => undefined,
     logger: {
@@ -103,7 +103,7 @@ async function setup(opts: {
     changeHandler,
     getSystemHealthStatus,
     broadcast,
-    invalidateSystem,
+    reconcile,
   };
 }
 
@@ -162,12 +162,21 @@ describe("setupRollupConsumer subscription", () => {
 });
 
 describe("setupRollupConsumer rollup recompute", () => {
-  it("broadcasts SYSTEM_STATUS_CHANGED + invalidates cache on a rollup status change", async () => {
+  it("reconciles the cache + broadcasts SYSTEM_STATUS_CHANGED on a rollup status change", async () => {
     const h = await setup({ statuses: ["healthy", "unhealthy"] });
     await h.consumeHandler({ data: { systemId: "s1" } });
 
     expect(h.getSystemHealthStatus).toHaveBeenCalled();
-    expect(h.invalidateSystem).toHaveBeenCalledWith("s1");
+    // Reconcile is handed the FULL prev/next states; the change-gating (evict +
+    // cluster broadcast only on a per-check vector change) lives inside the cache
+    // (see cache.test.ts), so recompute always calls it with both states.
+    expect(h.reconcile).toHaveBeenCalledTimes(1);
+    expect(h.reconcile.mock.calls[0]![0]).toMatchObject({
+      systemId: "s1",
+      previous: { status: "healthy" },
+      next: { status: "unhealthy" },
+    });
+    // The frontend signal IS gated here on the rollup-enum transition.
     expect(h.broadcast).toHaveBeenCalledTimes(1);
     const payload = h.broadcast.mock.calls[0]![1] as {
       systemId: string;
@@ -181,11 +190,13 @@ describe("setupRollupConsumer rollup recompute", () => {
     });
   });
 
-  it("does NOT broadcast when the rollup status is unchanged", async () => {
+  it("does NOT broadcast the frontend signal when the rollup status is unchanged", async () => {
     const h = await setup({ statuses: ["degraded", "degraded"] });
     await h.consumeHandler({ data: { systemId: "s1" } });
 
+    // No enum transition ⇒ no SYSTEM_STATUS_CHANGED frontend signal. Reconcile is
+    // still invoked (its own fingerprint gate no-ops for an unchanged vector).
     expect(h.broadcast).not.toHaveBeenCalled();
-    expect(h.invalidateSystem).not.toHaveBeenCalled();
+    expect(h.reconcile).toHaveBeenCalledTimes(1);
   });
 });