@checkstack/healthcheck-backend 1.18.0 → 1.20.0

package/src/cache.ts

@@ -1,57 +1,145 @@
 import type { CacheManager } from "@checkstack/cache-api";
-import {
-  createCachedScope,
-  type CachedScope,
-} from "@checkstack/cache-utils";
+import { createCachedScope, type CachedScope } from "@checkstack/cache-utils";
 import type { Logger } from "@checkstack/backend-api";
-import type { HealthCheckService } from "./service";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
+import { statusVectorChanged } from "./status-fingerprint";
 
 /**
- * TTL chosen to be slightly shorter than the dashboard's 30s `staleTime` so
- * that signal-driven invalidation almost always wins, and TTL only acts as
- * a safety net for paths that forget to invalidate.
+ * TTL for a cached status entry. With a distributed backend (Redis) configured,
+ * the TTL is only a natural refresh / safety net — cross-pod coherence comes
+ * from the SHARED store (a `delete` on one pod is visible to all), not from the
+ * TTL. On the default in-memory backend the entry is per-pod; that backend is
+ * for single-instance deployments (see the caching-architecture docs).
  */
 const STATUS_TTL_MS = 15_000;
 
+const STATUS_KEY_PREFIX = "status:";
+
+/**
+ * Per-(system, environment) cache key.
+ *
+ * `environmentId` collapses `undefined` (the system ROLLUP, all runs) and `null`
+ * (the env-less slice) to the SAME bare key `status:<systemId>`, because an
+ * env-less run IS the rollup (it mutates the bare `<systemId>` entity). A real
+ * environment id gets its own `status:<systemId>:<environmentId>` key. This
+ * matches the (systemId, environmentId) tuple both the reader
+ * (`getSystemHealthStatus`) and the executor's per-env write use.
+ */
+function statusKey(systemId: string, environmentId?: string | null): string {
+  return environmentId === undefined || environmentId === null
+    ? `${STATUS_KEY_PREFIX}${systemId}`
+    : `${STATUS_KEY_PREFIX}${systemId}:${environmentId}`;
+}
+
+/** Prefix covering a single system's rollup key AND all its per-env keys. */
+function systemPrefix(systemId: string): string {
+  return `${STATUS_KEY_PREFIX}${systemId}`;
+}
+
 /**
- * Per-entity cache helpers for the healthcheck plugin. Wrapping reads
- * goes through {@link wrapSystemHealthStatus}; mutations should call
- * {@link invalidateSystem} after the DB write but before emitting any
- * signal so that frontend refetches see fresh data.
+ * Minimal read surface the cache needs. `HealthCheckService` satisfies it
+ * structurally; a narrow interface keeps the cache testable with a stub and
+ * documents that the cache is the ONLY sanctioned caller of the raw
+ * `getSystemHealthStatus` read (enforced by the `no-direct-system-status-read`
+ * lint rule everywhere except this module and the executor/entity compute
+ * paths).
+ */
+export interface HealthStatusReader {
+  getSystemHealthStatus(
+    systemId: string,
+    environmentId?: string | null,
+  ): Promise<SystemHealthStatusResponse>;
+  /** Distinct environment ids a system currently has runs for (env-less excluded). */
+  getSystemEnvironmentIds(systemId: string): Promise<string[]>;
+}
+
+/** Per-(system, check, environment) matrix — see {@link HealthCheckCache.readMatrix}. */
+export type SystemHealthMatrix = Record<
+  string,
+  {
+    status: SystemHealthStatusResponse["status"];
+    checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+    environments: Record<
+      string,
+      {
+        status: SystemHealthStatusResponse["status"];
+        checkStatuses: SystemHealthStatusResponse["checkStatuses"];
+      }
+    >;
+  }
+>;
+
+/**
+ * The system-health status cache — a platform interface that is the SINGLE
+ * sanctioned reader AND invalidator of a system's derived health status.
+ *
+ * It is built on the platform {@link CacheManager} (via {@link createCachedScope}),
+ * so the active backend is a per-deployment choice: the default in-memory
+ * backend (per-pod, single-instance) or a distributed backend such as Redis
+ * (shared across pods). Cross-pod coherence therefore comes from the SHARED
+ * store — an `invalidate` is a `delete` every pod sees — NOT from any
+ * application-level broadcast. Horizontal scaling requires a distributed backend
+ * (see `docs/.../architecture/caching.md`).
+ *
+ * Reads (`read` / `readBulk` / `readMatrix`) serve the RAW (pre-incident-override)
+ * status; the router folds incident overrides downstream, so an incident change
+ * never touches this cache. `reconcile` is the hot-path invalidator: it evicts
+ * ONLY when the per-check status vector actually changed ({@link statusVectorChanged}),
+ * so a run that merely refreshes timestamps keeps the cache warm.
  */
 export interface HealthCheckCache {
-  /** Read-through cache for one system's health status. */
-  wrapSystemHealthStatus: (
+  /** Read-through cache for one (system, environment) status (RAW, pre-override). */
+  read(
     systemId: string,
-    loader: () => ReturnType<HealthCheckService["getSystemHealthStatus"]>,
-  ) => ReturnType<HealthCheckService["getSystemHealthStatus"]>;
+    environmentId?: string,
+  ): Promise<SystemHealthStatusResponse>;
 
-  /** Invalidate a single system's cached status. */
-  invalidateSystem: (systemId: string) => Promise<void>;
+  /** Per-entity read-through cache for many systems' rollup status. */
+  readBulk(
+    systemIds: string[],
+    environmentId?: string,
+  ): Promise<Record<string, SystemHealthStatusResponse>>;
 
   /**
-   * Invalidate every system's cached status. Used when the change can
-   * affect many systems at once (e.g. a configuration update with
-   * cross-system fan-out, or a plugin reload).
+   * Per-(system, check, environment) matrix assembled from cached rollup +
+   * per-environment reads. The env set is discovered live per system; each
+   * slice read hits the same per-env cache the badge path warms.
    */
-  invalidateAllSystems: () => Promise<number>;
+  readMatrix(systemIds: string[]): Promise<SystemHealthMatrix>;
 
-  /** Underlying scope, exposed for advanced callers. */
-  scope: CachedScope;
-}
+  /**
+   * Change-gated invalidator for the run hot path. Evicts the (system,
+   * environment) key ONLY when the derived status vector changed between
+   * `previous` and `next`; a no-op otherwise so a timestamp-only run keeps the
+   * cache warm. A per-environment change also evicts the system rollup key
+   * (the slice feeds the worst-wins rollup).
+   */
+  reconcile(args: {
+    systemId: string;
+    /** `null`/absent = the rollup / env-less key; a string = that environment. */
+    environmentId?: string | null;
+    previous: SystemHealthStatusResponse;
+    next: SystemHealthStatusResponse;
+  }): Promise<void>;
 
-const STATUS_KEY_PREFIX = "status:";
-const statusKey = (systemId: string): string =>
-  `${STATUS_KEY_PREFIX}${systemId}`;
+  /** Evict a system's rollup + every per-env key. */
+  invalidateSystem(systemId: string): Promise<void>;
+
+  /** Evict every system's status. Returns keys removed. */
+  invalidateAllSystems(): Promise<number>;
+}
 
 export function createHealthCheckCache({
   cacheManager,
   logger,
+  service,
 }: {
   cacheManager: CacheManager;
   logger: Logger;
+  /** Read source for cache misses. `HealthCheckService` satisfies this. */
+  service: HealthStatusReader;
 }): HealthCheckCache {
-  const scope = createCachedScope({
+  const scope: CachedScope = createCachedScope({
     cacheManager,
     pluginId: "healthcheck",
     defaultTtlMs: STATUS_TTL_MS,
@@ -60,11 +148,93 @@ export function createHealthCheckCache({
     },
   });
 
+  const read: HealthCheckCache["read"] = (systemId, environmentId) =>
+    scope.wrap(statusKey(systemId, environmentId), () =>
+      service.getSystemHealthStatus(systemId, environmentId),
+    );
+
+  const readBulk: HealthCheckCache["readBulk"] = async (
+    systemIds,
+    environmentId,
+  ) => {
+    const values = await scope.wrapMany(systemIds, {
+      keyFor: (id) => statusKey(id, environmentId),
+      loader: (id) => service.getSystemHealthStatus(id, environmentId),
+    });
+    const out: Record<string, SystemHealthStatusResponse> = {};
+    for (const [i, id] of systemIds.entries()) {
+      out[id] = values[i]!;
+    }
+    return out;
+  };
+
+  const readMatrix: HealthCheckCache["readMatrix"] = async (systemIds) => {
+    const result: SystemHealthMatrix = {};
+    await Promise.all(
+      systemIds.map(async (systemId) => {
+        const overall = await read(systemId);
+        const envIds = await service.getSystemEnvironmentIds(systemId);
+        const environments: SystemHealthMatrix[string]["environments"] = {};
+        await Promise.all(
+          envIds.map(async (environmentId) => {
+            const slice = await read(systemId, environmentId);
+            environments[environmentId] = {
+              status: slice.status,
+              checkStatuses: slice.checkStatuses,
+            };
+          }),
+        );
+        result[systemId] = {
+          status: overall.status,
+          checkStatuses: overall.checkStatuses,
+          environments,
+        };
+      }),
+    );
+    return result;
+  };
+
+  const reconcile: HealthCheckCache["reconcile"] = async ({
+    systemId,
+    environmentId,
+    previous,
+    next,
+  }) => {
+    if (!statusVectorChanged(previous, next)) return; // vector unchanged: keep warm.
+    // `scope.invalidate` is a `delete` on the active backend. With a distributed
+    // backend that delete is visible to every pod immediately, so no broadcast
+    // is needed for cross-pod coherence.
+    const isEnvScoped =
+      environmentId !== undefined && environmentId !== null;
+    if (isEnvScoped) {
+      // A per-environment slice changed. Evict its key AND the system rollup:
+      // the slice feeds the worst-wins rollup, so the rollup value may have
+      // moved even when its OWN per-check fingerprint can't see it (one slice
+      // recovering as another fails keeps `failingSliceCount` put). Sibling env
+      // keys stay warm.
+      await scope.invalidate(statusKey(systemId, environmentId));
+      await scope.invalidate(statusKey(systemId));
+    } else {
+      // Env-less / rollup change: the bare key IS the rollup.
+      await scope.invalidate(statusKey(systemId));
+    }
+  };
+
+  const invalidateSystem: HealthCheckCache["invalidateSystem"] = async (
+    systemId,
+  ) => {
+    await scope.invalidatePrefix(systemPrefix(systemId));
+  };
+
+  const invalidateAllSystems: HealthCheckCache["invalidateAllSystems"] = () =>
+    scope.invalidatePrefix(STATUS_KEY_PREFIX);
+
   return {
-    wrapSystemHealthStatus: (systemId, loader) =>
-      scope.wrap(statusKey(systemId), loader),
-    invalidateSystem: (systemId) => scope.invalidate(statusKey(systemId)),
-    invalidateAllSystems: () => scope.invalidatePrefix(STATUS_KEY_PREFIX),
-    scope,
+    read,
+    readBulk,
+    readMatrix,
+    reconcile,
+    invalidateSystem,
+    invalidateAllSystems,
   };
 }

package/src/health-notification-content.test.ts

@@ -0,0 +1,111 @@
+import { describe, it, expect } from "bun:test";
+import { buildHealthTransitionNotification } from "./health-notification-content";
+
+describe("buildHealthTransitionNotification", () => {
+  const base = {
+    systemId: "sys-1",
+    systemName: "Payments API",
+    configurationId: "cfg-9",
+    checkName: "HTTP 200 probe",
+    newStatus: "unhealthy" as const,
+  };
+
+  it("names the failing check in the body for an unhealthy transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+    });
+    expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
+    expect(payload.body).toContain("**Payments API**");
+    expect(payload.importance).toBe("critical");
+  });
+
+  it("names the failing check for a degraded transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      newStatus: "degraded",
+      transition: "escalation",
+    });
+    expect(payload.body).toContain('Health check **"HTTP 200 probe"**');
+    expect(payload.importance).toBe("warning");
+  });
+
+  it("pushes a healthcheck.healthcheck subject alongside the system subject", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+    });
+    const subjects = payload.subjects ?? [];
+    expect(subjects).toHaveLength(2);
+    expect(subjects[0]).toMatchObject({
+      kind: "catalog.system",
+      id: "sys-1",
+      name: "Payments API",
+    });
+    expect(subjects[1]).toMatchObject({
+      kind: "healthcheck.healthcheck",
+      id: "cfg-9",
+      name: "HTTP 200 probe",
+      status: "unhealthy",
+    });
+    // Check subject deep-links to its run history.
+    expect(subjects[1]?.url).toContain("sys-1");
+    expect(subjects[1]?.url).toContain("cfg-9");
+  });
+
+  it("falls back to the configuration id when no name is resolved", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      checkName: "cfg-9",
+      transition: "escalation",
+    });
+    expect(payload.body).toContain('Health check **"cfg-9"**');
+    expect((payload.subjects ?? [])[1]).toMatchObject({ name: "cfg-9" });
+  });
+
+  it("qualifies the body with the environment name when env-scoped", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+      environmentId: "env-prod",
+      environmentName: "Production",
+    });
+    expect(payload.body).toContain("in environment **Production**");
+    expect(payload.title).toContain("(Production)");
+  });
+
+  it("carries the origin environment id for an env-scoped transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+      environmentId: "env-prod",
+      environmentName: "Production",
+    });
+    // The status-page fan-out reads this to drop the change for a page that does
+    // not publish env-prod.
+    expect(payload.originEnvironmentId).toBe("env-prod");
+  });
+
+  it("omits the origin environment id for a system-rollup transition", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      transition: "escalation",
+    });
+    // No environmentId => a whole-system rollup; it must reach every page that
+    // surfaces the system regardless of environment.
+    expect(payload.originEnvironmentId).toBeUndefined();
+  });
+
+  it("stays system-level and omits the check subject on recovery", () => {
+    const payload = buildHealthTransitionNotification({
+      ...base,
+      newStatus: "healthy",
+      transition: "recovery",
+    });
+    expect(payload.body).not.toContain("Health check **");
+    expect(payload.importance).toBe("info");
+    const subjects = payload.subjects ?? [];
+    expect(subjects).toHaveLength(1);
+    expect(subjects[0]).toMatchObject({ kind: "catalog.system" });
+  });
+});

package/src/health-notification-content.ts

@@ -0,0 +1,145 @@
+import { resolveRoute, type InferClient } from "@checkstack/common";
+import { catalogRoutes, createSystemSubject } from "@checkstack/catalog-common";
+import type { NotificationApi } from "@checkstack/notification-common";
+import {
+  createHealthcheckSubject,
+  healthcheckRoutes,
+  systemHealthCollapseKey,
+  healthcheckSystemSubscription,
+  type HealthCheckStatus,
+} from "@checkstack/healthcheck-common";
+import type { TransitionKind } from "./notification-policy";
+
+/** The subset of `notifyForSubscription`'s input this builder produces. */
+type NotifyForSubscriptionInput = Parameters<
+  InferClient<typeof NotificationApi>["notifyForSubscription"]
+>[0];
+
+/**
+ * Inputs to {@link buildHealthTransitionNotification}. Pure data only - the
+ * catalog client is unused here (parents are resolved server-side) and thus
+ * omitted; every field is derived before the call site in the queue executor.
+ */
+export interface HealthTransitionNotificationInput {
+  transition: Exclude<TransitionKind, "none">;
+  systemId: string;
+  systemName: string;
+  configurationId: string;
+  /** Resolved display name of the check that drove the transition. */
+  checkName: string;
+  newStatus: HealthCheckStatus;
+  /** Concrete env id for a per-env slice, null/undefined for the system rollup. */
+  environmentId?: string | null;
+  /** Human-readable env name for the body/title. */
+  environmentName?: string;
+}
+
+/**
+ * Build the notification payload for a health-state transition. Pure and
+ * side-effect free so it can be unit-tested directly. Extracted from
+ * `notifyStateChange` so the body/title/subject wording (which now NAMES the
+ * failing check and pushes a `healthcheck.healthcheck` subject) is verifiable
+ * without booting the whole queue executor.
+ *
+ * Recovery bodies stay system-level (the whole system is green again; naming
+ * one check would mislead) and omit the check subject. Failing transitions
+ * (escalation / de-escalation) name the check in the body and add it as a
+ * subject deep-linked to its run history.
+ */
+export function buildHealthTransitionNotification(
+  input: HealthTransitionNotificationInput,
+): NotifyForSubscriptionInput {
+  const {
+    transition,
+    systemId,
+    systemName,
+    configurationId,
+    checkName,
+    newStatus,
+    environmentId,
+    environmentName,
+  } = input;
+
+  const envScoped = typeof environmentId === "string";
+  const envSuffix = envScoped && environmentName ? ` (${environmentName})` : "";
+  const envQualifier = envScoped
+    ? ` in environment **${environmentName ?? environmentId}**`
+    : "";
+
+  let title: string;
+  let body: string;
+  let importance: "info" | "warning" | "critical";
+
+  if (transition === "recovery") {
+    title = `System health restored${envSuffix}: ${systemName}`;
+    body = envScoped
+      ? `Health checks for **${systemName}** in environment **${environmentName ?? environmentId}** are now passing. The system has returned to normal operation in that environment.`
+      : `All health checks for **${systemName}** are now passing. The system has returned to normal operation.`;
+    importance = "info";
+  } else if (newStatus === "unhealthy") {
+    title = `System health critical${envSuffix}: ${systemName}`;
+    body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system is unhealthy and may be down${envScoped ? " in that environment" : ""}.`;
+    importance = "critical";
+  } else {
+    // degraded - either an escalation from healthy or a partial recovery
+    title = `System health degraded${envSuffix}: ${systemName}`;
+    body = `Health check **"${checkName}"** on **${systemName}**${envQualifier} is failing. The system may be experiencing issues${envScoped ? " in that environment" : ""}.`;
+    importance = "warning";
+  }
+
+  const systemDetailPath = resolveRoute(catalogRoutes.routes.systemDetail, {
+    systemId,
+  });
+  // Recovery lands on the default (all) view; failing transitions deep-link
+  // operators into the failing-checks filter so they can debug immediately.
+  const actionUrl =
+    transition === "recovery"
+      ? systemDetailPath
+      : `${systemDetailPath}?filter=failing`;
+  const actionLabel =
+    transition === "recovery" ? "View System" : "View failing checks";
+
+  return {
+    specId: healthcheckSystemSubscription.specId,
+    resourceKeys: [systemId],
+    title,
+    body,
+    importance,
+    action: { label: actionLabel, url: actionUrl },
+    // Carry the failing ENVIRONMENT so the status-page fan-out can drop this
+    // change for a page that does not publish that environment (e.g. a
+    // `development` failure never reaches a prod-only page's subscribers). Only
+    // set for an env-scoped slice; the system-rollup path stays env-less.
+    ...(typeof environmentId === "string"
+      ? { originEnvironmentId: environmentId }
+      : {}),
+    // Env-qualified collapse key so two failing envs of one system generate
+    // two independent notification cards (one per env) instead of merging.
+    collapseKey: envScoped
+      ? systemHealthCollapseKey(systemId, environmentId)
+      : systemHealthCollapseKey(systemId),
+    subjects: [
+      createSystemSubject({
+        id: systemId,
+        name: systemName,
+        url: systemDetailPath,
+        status: newStatus,
+      }),
+      // Name the failing check as its own subject for every non-recovery
+      // transition, deep-linked to its run history. Omitted on recovery.
+      ...(transition === "recovery"
+        ? []
+        : [
+            createHealthcheckSubject({
+              id: configurationId,
+              name: checkName,
+              url: resolveRoute(healthcheckRoutes.routes.historyDetail, {
+                systemId,
+                configurationId,
+              }),
+              status: newStatus,
+            }),
+          ]),
+    ],
+  };
+}

package/src/healthcheck-gitops-kinds.test.ts

@@ -5,6 +5,7 @@ import {
   buildHealthcheckKind,
   buildSystemHealthcheckExtension,
 } from "./healthcheck-gitops-kinds";
+import { createStubHealthCheckCache } from "./cache-test-stub";
 import type {
   HealthCheckConfiguration,
   CreateHealthCheckConfiguration,
@@ -248,10 +249,13 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
   let mockHCRegistry: ReturnType<typeof createMockHealthCheckRegistry>;
   let mockCollectorRegistry: ReturnType<typeof createMockCollectorRegistry>;
 
+  let mockCache: ReturnType<typeof createStubHealthCheckCache>;
+
   beforeEach(() => {
     mockService = createMockService();
     mockHCRegistry = createMockHealthCheckRegistry();
     mockCollectorRegistry = createMockCollectorRegistry();
+    mockCache = createStubHealthCheckCache();
   });
 
   function buildKind() {
@@ -262,6 +266,7 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
       getQueueManager: () => emptyReconcileQueueManager(),
       getDb: () => emptyReconcileDbStub(),
       getCatalogClient: () => emptyCatalogClient(),
+      getCache: () => mockCache,
     };
     return buildHealthcheckKind(mockDeps);
   }
@@ -601,6 +606,9 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
 
     expect(result.entityId).toBe("hc-1");
     expect(mockService.createConfiguration).toHaveBeenCalledTimes(1);
+    // Regression guard: GitOps must invalidate the status cache (a create could
+    // affect any system's rollup), otherwise every pod serves stale health.
+    expect(mockCache.invalidateAllSystems).toHaveBeenCalledTimes(1);
   });
 });
 
@@ -608,9 +616,11 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
 
 describe("Healthcheck GitOps Kind: System Extension", () => {
   let mockService: ReturnType<typeof createMockService>;
+  let mockCache: ReturnType<typeof createStubHealthCheckCache>;
 
   beforeEach(() => {
     mockService = createMockService();
+    mockCache = createStubHealthCheckCache();
   });
 
   function buildExtension() {
@@ -627,6 +637,7 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
       getQueueManager: () => emptyReconcileQueueManager(),
       getDb: () => emptyReconcileDbStub(),
       getCatalogClient: () => emptyCatalogClient(),
+      getCache: () => mockCache,
     });
   }
 
@@ -667,6 +678,9 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
     expect(mockService.associations[0].systemId).toBe("sys-123");
     expect(mockService.associations[0].configurationId).toBe("hc-1");
     expect(mockService.associations[1].configurationId).toBe("hc-2");
+    // Regression guard: an association change must invalidate THIS system's
+    // cached status (rollup + env keys), else pods serve stale health.
+    expect(mockCache.invalidateSystem).toHaveBeenCalledWith("sys-123");
   });
 
   it("removes stale associations not in spec", async () => {

package/src/healthcheck-gitops-kinds.ts

@@ -15,6 +15,7 @@ import type {
 } from "@checkstack/backend-api";
 import { NotificationPolicySchema } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
+import type { HealthCheckCache } from "./cache";
 import { validateVersionedConfigStrict } from "./validate-configuration";
 import {
   DynamicOperators,
@@ -43,6 +44,15 @@ interface HealthcheckGitOpsKindsDeps {
   getQueueManager: () => QueueManager;
   getDb: () => SafeDatabase<typeof schema>;
   getCatalogClient: () => InferClient<typeof CatalogApi>;
+  /**
+   * Lazy accessor for the system-health status cache. GitOps writes configs /
+   * associations directly on the service (not through the router), so they MUST
+   * invalidate the cache themselves — otherwise a `git push` that changes a
+   * system's derived status leaves every pod serving the stale cached value
+   * until its 15s TTL. Mirrors the router's config-mutation invalidation. May be
+   * `undefined` before init completes (reconcile only runs post-init).
+   */
+  getCache: () => HealthCheckCache | undefined;
 }
 
 // ─── Healthcheck Spec Schema ───────────────────────────────────────────────
@@ -264,6 +274,11 @@ export function buildHealthcheckKind(
           // silently restored from the stored row (keep-existing is UI-only).
           { mergeSecrets: false },
         );
+        // A config change (thresholds, assertions, interval) can move the
+        // derived status of every system this check is assigned to. Match the
+        // router's `updateConfiguration` handler: drop every system's cached
+        // status (+ broadcast) so no pod serves stale health.
+        await deps.getCache()?.invalidateAllSystems();
         context.logger.info(
           `GitOps: updated Healthcheck "${displayName}" (id: ${existingEntityId})`,
         );
@@ -282,6 +297,9 @@ export function buildHealthcheckKind(
           assertions: c.assertions,
         })),
       });
+      // A new configuration could be associated with any system; match the
+      // router's `createConfiguration` handler and drop every cached status.
+      await deps.getCache()?.invalidateAllSystems();
       context.logger.info(
         `GitOps: created Healthcheck "${displayName}" (id: ${config.id})`,
       );
@@ -299,6 +317,9 @@ export function buildHealthcheckKind(
       if (!entityId) return;
       const service = deps.createService();
       await service.deleteConfiguration(entityId);
+      // Match the router's `deleteConfiguration` handler: drop every cached
+      // status since the removed check could have affected any system's rollup.
+      await deps.getCache()?.invalidateAllSystems();
       context.logger.info(`GitOps: deleted Healthcheck (id: ${entityId})`);
     },
   };
@@ -415,6 +436,12 @@ export function buildSystemHealthcheckExtension(
           );
         }
       }
+
+      // Every association add/remove above changes THIS system's derived
+      // rollup (a check enters/leaves worst-wins), so drop its cached status
+      // (rollup + all env keys) and broadcast — mirroring the router's
+      // associate/disassociate handlers.
+      await deps.getCache()?.invalidateSystem(systemEntityId);
     },
   };
 }