@checkstack/healthcheck-backend 1.19.0 → 1.20.1

package/src/cache.ts

@@ -1,57 +1,145 @@
 import type { CacheManager } from "@checkstack/cache-api";
-import {
-  createCachedScope,
-  type CachedScope,
-} from "@checkstack/cache-utils";
+import { createCachedScope, type CachedScope } from "@checkstack/cache-utils";
 import type { Logger } from "@checkstack/backend-api";
-import type { HealthCheckService } from "./service";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
+import { statusVectorChanged } from "./status-fingerprint";
 
 /**
- * TTL chosen to be slightly shorter than the dashboard's 30s `staleTime` so
- * that signal-driven invalidation almost always wins, and TTL only acts as
- * a safety net for paths that forget to invalidate.
+ * TTL for a cached status entry. With a distributed backend (Redis) configured,
+ * the TTL is only a natural refresh / safety net — cross-pod coherence comes
+ * from the SHARED store (a `delete` on one pod is visible to all), not from the
+ * TTL. On the default in-memory backend the entry is per-pod; that backend is
+ * for single-instance deployments (see the caching-architecture docs).
  */
 const STATUS_TTL_MS = 15_000;
 
+const STATUS_KEY_PREFIX = "status:";
+
+/**
+ * Per-(system, environment) cache key.
+ *
+ * `environmentId` collapses `undefined` (the system ROLLUP, all runs) and `null`
+ * (the env-less slice) to the SAME bare key `status:<systemId>`, because an
+ * env-less run IS the rollup (it mutates the bare `<systemId>` entity). A real
+ * environment id gets its own `status:<systemId>:<environmentId>` key. This
+ * matches the (systemId, environmentId) tuple both the reader
+ * (`getSystemHealthStatus`) and the executor's per-env write use.
+ */
+function statusKey(systemId: string, environmentId?: string | null): string {
+  return environmentId === undefined || environmentId === null
+    ? `${STATUS_KEY_PREFIX}${systemId}`
+    : `${STATUS_KEY_PREFIX}${systemId}:${environmentId}`;
+}
+
+/** Prefix covering a single system's rollup key AND all its per-env keys. */
+function systemPrefix(systemId: string): string {
+  return `${STATUS_KEY_PREFIX}${systemId}`;
+}
+
 /**
- * Per-entity cache helpers for the healthcheck plugin. Wrapping reads
- * goes through {@link wrapSystemHealthStatus}; mutations should call
- * {@link invalidateSystem} after the DB write but before emitting any
- * signal so that frontend refetches see fresh data.
+ * Minimal read surface the cache needs. `HealthCheckService` satisfies it
+ * structurally; a narrow interface keeps the cache testable with a stub and
+ * documents that the cache is the ONLY sanctioned caller of the raw
+ * `getSystemHealthStatus` read (enforced by the `no-direct-system-status-read`
+ * lint rule everywhere except this module and the executor/entity compute
+ * paths).
+ */
+export interface HealthStatusReader {
+  getSystemHealthStatus(
+    systemId: string,
+    environmentId?: string | null,
+  ): Promise<SystemHealthStatusResponse>;
+  /** Distinct environment ids a system currently has runs for (env-less excluded). */
+  getSystemEnvironmentIds(systemId: string): Promise<string[]>;
+}
+
+/** Per-(system, check, environment) matrix — see {@link HealthCheckCache.readMatrix}. */
+export type SystemHealthMatrix = Record<
+  string,
+  {
+    status: SystemHealthStatusResponse["status"];
+    checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+    environments: Record<
+      string,
+      {
+        status: SystemHealthStatusResponse["status"];
+        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+      }
+    >;
+  }
+>;
+
+/**
+ * The system-health status cache — a platform interface that is the SINGLE
+ * sanctioned reader AND invalidator of a system's derived health status.
+ *
+ * It is built on the platform {@link CacheManager} (via {@link createCachedScope}),
+ * so the active backend is a per-deployment choice: the default in-memory
+ * backend (per-pod, single-instance) or a distributed backend such as Redis
+ * (shared across pods). Cross-pod coherence therefore comes from the SHARED
+ * store — an `invalidate` is a `delete` every pod sees — NOT from any
+ * application-level broadcast. Horizontal scaling requires a distributed backend
+ * (see `docs/.../architecture/caching.md`).
+ *
+ * Reads (`read` / `readBulk` / `readMatrix`) serve the RAW (pre-incident-override)
+ * status; the router folds incident overrides downstream, so an incident change
+ * never touches this cache. `reconcile` is the hot-path invalidator: it evicts
+ * ONLY when the per-check status vector actually changed ({@link statusVectorChanged}),
+ * so a run that merely refreshes timestamps keeps the cache warm.
  */
 export interface HealthCheckCache {
-  /** Read-through cache for one system's health status. */
-  wrapSystemHealthStatus: (
+  /** Read-through cache for one (system, environment) status (RAW, pre-override). */
+  read(
     systemId: string,
-    loader: () => ReturnType<HealthCheckService["getSystemHealthStatus"]>,
-  ) => ReturnType<HealthCheckService["getSystemHealthStatus"]>;
+    environmentId?: string,
+  ): Promise<SystemHealthStatusResponse>;
 
-  /** Invalidate a single system's cached status. */
-  invalidateSystem: (systemId: string) => Promise<void>;
+  /** Per-entity read-through cache for many systems' rollup status. */
+  readBulk(
+    systemIds: string[],
+    environmentId?: string,
+  ): Promise<Record<string, SystemHealthStatusResponse>>;
 
   /**
-   * Invalidate every system's cached status. Used when the change can
-   * affect many systems at once (e.g. a configuration update with
-   * cross-system fan-out, or a plugin reload).
+   * Per-(system, check, environment) matrix assembled from cached rollup +
+   * per-environment reads. The env set is discovered live per system; each
+   * slice read hits the same per-env cache the badge path warms.
    */
-  invalidateAllSystems: () => Promise<number>;
+  readMatrix(systemIds: string[]): Promise<SystemHealthMatrix>;
 
-  /** Underlying scope, exposed for advanced callers. */
-  scope: CachedScope;
-}
+  /**
+   * Change-gated invalidator for the run hot path. Evicts the (system,
+   * environment) key ONLY when the derived status vector changed between
+   * `previous` and `next`; a no-op otherwise so a timestamp-only run keeps the
+   * cache warm. A per-environment change also evicts the system rollup key
+   * (the slice feeds the worst-wins rollup).
+   */
+  reconcile(args: {
+    systemId: string;
+    /** `null`/absent = the rollup / env-less key; a string = that environment. */
+    environmentId?: string | null;
+    previous: SystemHealthStatusResponse;
+    next: SystemHealthStatusResponse;
+  }): Promise<void>;
 
-const STATUS_KEY_PREFIX = "status:";
-const statusKey = (systemId: string): string =>
-  `${STATUS_KEY_PREFIX}${systemId}`;
+  /** Evict a system's rollup + every per-env key. */
+  invalidateSystem(systemId: string): Promise<void>;
+
+  /** Evict every system's status. Returns keys removed. */
+  invalidateAllSystems(): Promise<number>;
+}
 
 export function createHealthCheckCache({
   cacheManager,
   logger,
+  service,
 }: {
   cacheManager: CacheManager;
   logger: Logger;
+  /** Read source for cache misses. `HealthCheckService` satisfies this. */
+  service: HealthStatusReader;
 }): HealthCheckCache {
-  const scope = createCachedScope({
+  const scope: CachedScope = createCachedScope({
     cacheManager,
     pluginId: "healthcheck",
     defaultTtlMs: STATUS_TTL_MS,
@@ -60,11 +148,93 @@ export function createHealthCheckCache({
     },
   });
 
+  const read: HealthCheckCache["read"] = (systemId, environmentId) =>
+    scope.wrap(statusKey(systemId, environmentId), () =>
+      service.getSystemHealthStatus(systemId, environmentId),
+    );
+
+  const readBulk: HealthCheckCache["readBulk"] = async (
+    systemIds,
+    environmentId,
+  ) => {
+    const values = await scope.wrapMany(systemIds, {
+      keyFor: (id) => statusKey(id, environmentId),
+      loader: (id) => service.getSystemHealthStatus(id, environmentId),
+    });
+    const out: Record<string, SystemHealthStatusResponse> = {};
+    for (const [i, id] of systemIds.entries()) {
+      out[id] = values[i]!;
+    }
+    return out;
+  };
+
+  const readMatrix: HealthCheckCache["readMatrix"] = async (systemIds) => {
+    const result: SystemHealthMatrix = {};
+    await Promise.all(
+      systemIds.map(async (systemId) => {
+        const overall = await read(systemId);
+        const envIds = await service.getSystemEnvironmentIds(systemId);
+        const environments: SystemHealthMatrix[string]["environments"] = {};
+        await Promise.all(
+          envIds.map(async (environmentId) => {
+            const slice = await read(systemId, environmentId);
+            environments[environmentId] = {
+              status: slice.status,
+              checkStatuses: slice.checkStatuses,
+            };
+          }),
+        );
+        result[systemId] = {
+          status: overall.status,
+          checkStatuses: overall.checkStatuses,
+          environments,
+        };
+      }),
+    );
+    return result;
+  };
+
+  const reconcile: HealthCheckCache["reconcile"] = async ({
+    systemId,
+    environmentId,
+    previous,
+    next,
+  }) => {
+    if (!statusVectorChanged(previous, next)) return; // vector unchanged: keep warm.
+    // `scope.invalidate` is a `delete` on the active backend. With a distributed
+    // backend that delete is visible to every pod immediately, so no broadcast
+    // is needed for cross-pod coherence.
+    const isEnvScoped =
+      environmentId !== undefined && environmentId !== null;
+    if (isEnvScoped) {
+      // A per-environment slice changed. Evict its key AND the system rollup:
+      // the slice feeds the worst-wins rollup, so the rollup value may have
+      // moved even when its OWN per-check fingerprint can't see it (one slice
+      // recovering as another fails keeps `failingSliceCount` put). Sibling env
+      // keys stay warm.
+      await scope.invalidate(statusKey(systemId, environmentId));
+      await scope.invalidate(statusKey(systemId));
+    } else {
+      // Env-less / rollup change: the bare key IS the rollup.
+      await scope.invalidate(statusKey(systemId));
+    }
+  };
+
+  const invalidateSystem: HealthCheckCache["invalidateSystem"] = async (
+    systemId,
+  ) => {
+    await scope.invalidatePrefix(systemPrefix(systemId));
+  };
+
+  const invalidateAllSystems: HealthCheckCache["invalidateAllSystems"] = () =>
+    scope.invalidatePrefix(STATUS_KEY_PREFIX);
+
   return {
-    wrapSystemHealthStatus: (systemId, loader) =>
-      scope.wrap(statusKey(systemId), loader),
-    invalidateSystem: (systemId) => scope.invalidate(statusKey(systemId)),
-    invalidateAllSystems: () => scope.invalidatePrefix(STATUS_KEY_PREFIX),
-    scope,
+    read,
+    readBulk,
+    readMatrix,
+    reconcile,
+    invalidateSystem,
+    invalidateAllSystems,
   };
 }

package/src/health-notification-content.test.ts

@@ -74,6 +74,28 @@ describe("buildHealthTransitionNotification", () => {
     expect(payload.title).toContain("(Production)");
   });
 
+  it("carries the origin environment id for an env-scoped transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+      environmentId: "env-prod",
+      environmentName: "Production",
+    });
+    // The status-page fan-out reads this to drop the change for a page that does
+    // not publish env-prod.
+    expect(payload.originEnvironmentId).toBe("env-prod");
+  });
+
+  it("omits the origin environment id for a system-rollup transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+    });
+    // No environmentId => a whole-system rollup; it must reach every page that
+    // surfaces the system regardless of environment.
+    expect(payload.originEnvironmentId).toBeUndefined();
+  });
+
   it("stays system-level and omits the check subject on recovery", () => {
     const payload = buildHealthTransitionNotification({
       ...base,

package/src/health-notification-content.ts

@@ -106,6 +106,13 @@ export function buildHealthTransitionNotification(
     body,
     importance,
     action: { label: actionLabel, url: actionUrl },
+    // Carry the failing ENVIRONMENT so the status-page fan-out can drop this
+    // change for a page that does not publish that environment (e.g. a
+    // `development` failure never reaches a prod-only page's subscribers). Only
+    // set for an env-scoped slice; the system-rollup path stays env-less.
+    ...(typeof environmentId === "string"
+      ? { originEnvironmentId: environmentId }
+      : {}),
     // Env-qualified collapse key so two failing envs of one system generate
     // two independent notification cards (one per env) instead of merging.
     collapseKey: envScoped

package/src/healthcheck-gitops-kinds.test.ts

@@ -5,6 +5,7 @@ import {
   buildHealthcheckKind,
   buildSystemHealthcheckExtension,
 } from "./healthcheck-gitops-kinds";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 import type {
   HealthCheckConfiguration,
   CreateHealthCheckConfiguration,
@@ -248,10 +249,13 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
   let mockHCRegistry: ReturnType<typeof createMockHealthCheckRegistry>;
   let mockCollectorRegistry: ReturnType<typeof createMockCollectorRegistry>;
 
+  let mockCache: ReturnType<typeof createStubHealthCheckCache>;
+
   beforeEach(() => {
     mockService = createMockService();
     mockHCRegistry = createMockHealthCheckRegistry();
     mockCollectorRegistry = createMockCollectorRegistry();
+    mockCache = createStubHealthCheckCache();
   });
 
   function buildKind() {
@@ -262,6 +266,7 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
       getQueueManager: () => emptyReconcileQueueManager(),
       getDb: () => emptyReconcileDbStub(),
       getCatalogClient: () => emptyCatalogClient(),
+      getCache: () => mockCache,
     };
     return buildHealthcheckKind(mockDeps);
   }
@@ -601,6 +606,9 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
 
     expect(result.entityId).toBe("hc-1");
     expect(mockService.createConfiguration).toHaveBeenCalledTimes(1);
+    // Regression guard: GitOps must invalidate the status cache (a create could
+    // affect any system's rollup), otherwise every pod serves stale health.
+    expect(mockCache.invalidateAllSystems).toHaveBeenCalledTimes(1);
   });
 });
 
@@ -608,9 +616,11 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
 
 describe("Healthcheck GitOps Kind: System Extension", () => {
   let mockService: ReturnType<typeof createMockService>;
+  let mockCache: ReturnType<typeof createStubHealthCheckCache>;
 
   beforeEach(() => {
     mockService = createMockService();
+    mockCache = createStubHealthCheckCache();
   });
 
   function buildExtension() {
@@ -627,6 +637,7 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
       getQueueManager: () => emptyReconcileQueueManager(),
       getDb: () => emptyReconcileDbStub(),
       getCatalogClient: () => emptyCatalogClient(),
+      getCache: () => mockCache,
     });
   }
 
@@ -667,6 +678,9 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
     expect(mockService.associations[0].systemId).toBe("sys-123");
     expect(mockService.associations[0].configurationId).toBe("hc-1");
     expect(mockService.associations[1].configurationId).toBe("hc-2");
+    // Regression guard: an association change must invalidate THIS system's
+    // cached status (rollup + env keys), else pods serve stale health.
+    expect(mockCache.invalidateSystem).toHaveBeenCalledWith("sys-123");
   });
 
   it("removes stale associations not in spec", async () => {

package/src/healthcheck-gitops-kinds.ts

@@ -15,6 +15,7 @@ import type {
 } from "@checkstack/backend-api";
 import { NotificationPolicySchema } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
+import type { HealthCheckCache } from "./cache";
 import { validateVersionedConfigStrict } from "./validate-configuration";
 import {
   DynamicOperators,
@@ -43,6 +44,15 @@ interface HealthcheckGitOpsKindsDeps {
   getQueueManager: () => QueueManager;
   getDb: () => SafeDatabase<typeof schema>;
   getCatalogClient: () => InferClient<typeof CatalogApi>;
+  /**
+   * Lazy accessor for the system-health status cache. GitOps writes configs /
+   * associations directly on the service (not through the router), so they MUST
+   * invalidate the cache themselves — otherwise a `git push` that changes a
+   * system's derived status leaves every pod serving the stale cached value
+   * until its 15s TTL. Mirrors the router's config-mutation invalidation. May be
+   * `undefined` before init completes (reconcile only runs post-init).
+   */
+  getCache: () => HealthCheckCache | undefined;
 }
 
 // ─── Healthcheck Spec Schema ───────────────────────────────────────────────
@@ -264,6 +274,11 @@ export function buildHealthcheckKind(
           // silently restored from the stored row (keep-existing is UI-only).
           { mergeSecrets: false },
         );
+        // A config change (thresholds, assertions, interval) can move the
+        // derived status of every system this check is assigned to. Match the
+        // router's `updateConfiguration` handler: drop every system's cached
+        // status (+ broadcast) so no pod serves stale health.
+        await deps.getCache()?.invalidateAllSystems();
         context.logger.info(
           `GitOps: updated Healthcheck "${displayName}" (id: ${existingEntityId})`,
         );
@@ -282,6 +297,9 @@ export function buildHealthcheckKind(
           assertions: c.assertions,
         })),
       });
+      // A new configuration could be associated with any system; match the
+      // router's `createConfiguration` handler and drop every cached status.
+      await deps.getCache()?.invalidateAllSystems();
       context.logger.info(
         `GitOps: created Healthcheck "${displayName}" (id: ${config.id})`,
       );
@@ -299,6 +317,9 @@ export function buildHealthcheckKind(
       if (!entityId) return;
       const service = deps.createService();
       await service.deleteConfiguration(entityId);
+      // Match the router's `deleteConfiguration` handler: drop every cached
+      // status since the removed check could have affected any system's rollup.
+      await deps.getCache()?.invalidateAllSystems();
       context.logger.info(`GitOps: deleted Healthcheck (id: ${entityId})`);
     },
   };
@@ -415,6 +436,12 @@ export function buildSystemHealthcheckExtension(
           );
         }
       }
+
+      // Every association add/remove above changes THIS system's derived
+      // rollup (a check enters/leaves worst-wins), so drop its cached status
+      // (rollup + all env keys) and broadcast — mirroring the router's
+      // associate/disassociate handlers.
+      await deps.getCache()?.invalidateSystem(systemEntityId);
     },
   };
 }

package/src/index.ts

@@ -230,6 +230,10 @@ export default createBackendPlugin({
           throw new Error("Catalog client not initialized");
         return gitopsCatalogClient;
       },
+      // Lazy: the cache is created in init() (after this register() call) and
+      // reconcile only runs post-init, so `healthCheckCache` is populated by the
+      // time GitOps mutations fire.
+      getCache: () => healthCheckCache,
     });
 
     env.registerInit({
@@ -461,18 +465,34 @@ export default createBackendPlugin({
           execute: deferredProjectionExecute,
         });
 
+        // Per-entity status cache shared between the router, queue executor,
+        // AI signals contributor, and afterPluginsReady cleanup hooks. It is the
+        // single sanctioned reader (compute-on-read via `service`) AND
+        // invalidator of a system's derived health status. It runs on the
+        // platform CacheManager, so cross-pod coherence comes from the SHARED
+        // backend (a distributed provider such as Redis makes an eviction
+        // visible to every pod); no application-level broadcast is needed.
+        const cache = createHealthCheckCache({
+          cacheManager,
+          logger,
+          service,
+        });
+        healthCheckCache = cache;
+
         // Contribute this plugin's per-system health problems to the AI
         // `system.issues` aggregator. PER-SOURCE access is OUR job: gate on the
         // principal's `healthcheck.status` grant and return {} (never throw)
-        // when not satisfied. The read derives from the durable
-        // `health_check_runs` / `system_health_checks` tables (global, identical
-        // on every pod) and reuses the SAME pure deriver as the dashboard
-        // filler, so backend signals match the UI's source/tone/label/detail.
+        // when not satisfied. The candidate set derives from the durable
+        // `system_health_checks` table (global, identical on every pod); the
+        // per-system status reads go through the SHARED CACHE (reusing the warm
+        // badge/dashboard entries) and reuse the SAME pure deriver as the
+        // dashboard filler, so backend signals match the UI.
         env
           .getExtensionPoint(systemSignalsExtensionPoint)
           .contribute(
             createHealthcheckSignalsContributor({
-              service,
+              candidateSource: service,
+              cache,
               resolver: createSystemAccessResolver(rpcClient),
             }),
           );
@@ -492,13 +512,6 @@ export default createBackendPlugin({
         // Create gitops client for provenance lock checks
         const gitOpsClient = rpcClient.forPlugin(GitOpsApi);
 
-        // Per-entity status cache shared between the router, queue executor,
-        // and afterPluginsReady cleanup hooks. Mutations / new check results
-        // invalidate by systemId BEFORE emitting signals so frontend
-        // refetches see fresh data.
-        const cache = createHealthCheckCache({ cacheManager, logger });
-        healthCheckCache = cache;
-
         // Setup queue-based health check worker
         await setupHealthCheckWorker({
           notificationClient,
@@ -630,6 +643,12 @@ export default createBackendPlugin({
       }) => {
         // Store emitHook for the queue worker (Closure-based Hook Getter pattern)
         storedEmitHook = emitHook;
+
+        // No cross-pod status-cache broadcast: the status cache runs on the
+        // platform CacheManager, so a distributed backend (Redis) makes every
+        // eviction visible to all pods through the shared store. The prior
+        // per-pod-cache + broadcast layer was removed in favour of that.
+
         // Converge the per-environment recurring job set at boot (schedule
         // desired (config, system, env) jobs, cancel orphans incl. old-format
         // ones). The periodic reconcile below keeps it converged as catalog

package/src/queue-executor.test.ts

@@ -5,16 +5,11 @@ import {
   recomputeSystemRollupHealth,
   type HealthCheckJobPayload,
 } from "./queue-executor";
-import type { HealthCheckCache } from "./cache";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 import { SuspectLane } from "./suspect-lane";
 import type { SlowCheckRuntime } from "./slow-check-config";
 
-const passthroughCache: HealthCheckCache = {
-  wrapSystemHealthStatus: (_systemId, loader) => loader(),
-  invalidateSystem: async () => {},
-  invalidateAllSystems: async () => 0,
-  scope: {} as HealthCheckCache["scope"],
-};
+const passthroughCache = createStubHealthCheckCache();
 
 // Pass-through advisory lock: these tests don't exercise cross-pod
 // serialization, so run the critical section directly.
@@ -248,16 +243,7 @@ describe("Queue-Based Health Check Executor", () => {
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
         if (selectCallCount === 1) {
-          // First call: get previous system health status
-          return {
-            from: mock(() => ({
-              innerJoin: mock(() => ({
-                where: mock(() => Promise.resolve([])),
-              })),
-            })),
-          };
-        } else if (selectCallCount === 2) {
-          // Second call: fetch configuration (return paused config)
+          // First call: fetch configuration (return paused config)
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -371,7 +357,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -592,7 +578,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock(() => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -762,7 +748,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock((...args: unknown[]) => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -1129,7 +1115,7 @@ describe("Queue-Based Health Check Executor", () => {
       let selectCallCount = 0;
       (mockDb.select as any) = mock((...args: unknown[]) => {
         selectCallCount++;
-        if (selectCallCount === 2) {
+        if (selectCallCount === 1) {
           return {
             from: mock(() => ({
               innerJoin: mock(() => ({
@@ -1323,7 +1309,7 @@ describe("executeHealthCheckJob - structured assertion outcomes", () => {
     let selectCallCount = 0;
     (mockDb.select as any) = mock(() => {
       selectCallCount++;
-      if (selectCallCount === 2) {
+      if (selectCallCount === 1) {
         return {
           from: mock(() => ({
             innerJoin: mock(() => ({
@@ -1534,13 +1520,14 @@ describe("executeHealthCheckJob - slow-check bulkhead wiring", () => {
     (mockCatalogClient.getSystem as any) = mock(async () => ({ id: "system-1", name: "web-01" }));
     (mockCatalogClient as any).resolveSystemEnvironments = mock(async () => []);
 
-    // Select call order: #1 getSystemHealthStatus (rollup prev), #2 config,
-    // #3 fetchRecentRunsForSlice. Everything else falls through to default.
+    // Select call order: #1 config, #2 fetchRecentRunsForSlice. Everything else
+    // falls through to default. (The eager rollup-prev read that used to be #1
+    // is now computed lazily only on the catastrophic-failure path.)
     const defaultSelect = mockDb.select;
     let selectCallCount = 0;
     (mockDb.select as any) = mock((...args: unknown[]) => {
       selectCallCount++;
-      if (selectCallCount === 2) {
+      if (selectCallCount === 1) {
         return {
           from: mock(() => ({
             innerJoin: mock(() => ({
@@ -1567,7 +1554,7 @@ describe("executeHealthCheckJob - slow-check bulkhead wiring", () => {
           })),
         };
       }
-      if (selectCallCount === 3) {
+      if (selectCallCount === 2) {
         return {
           from: mock(() => ({
             where: mock(() => ({