@checkstack/healthcheck-backend 1.17.0 → 1.19.0

package/src/index.ts

@@ -1,8 +1,8 @@
 import {
   setupHealthCheckWorker,
-  bootstrapHealthChecks,
   recomputeSystemRollupHealth,
 } from "./queue-executor";
+import { reconcileHealthCheckJobs } from "./schedule-reconciler";
 import { setupRetentionJob } from "./retention-job";
 import * as schema from "./schema";
 import {
@@ -38,6 +38,7 @@ import {
   type SafeDatabase,
   type HealthCheckRegistry,
   type CollectorRegistry,
+  type AdvisoryLockService,
 } from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
 import {
@@ -75,6 +76,9 @@ import { IncidentApi } from "@checkstack/incident-common";
 import { GitOpsApi } from "@checkstack/gitops-common";
 import { registerSearchProvider } from "@checkstack/command-backend";
 import { resolveRoute } from "@checkstack/common";
+import type { InferClient } from "@checkstack/common";
+import type { SignalService } from "@checkstack/signal-common";
+import { setupRollupConsumer } from "./rollup-consumer";
 import { createHealthCheckCache } from "./cache";
 import { inArray, ilike } from "drizzle-orm";
 
@@ -174,6 +178,12 @@ export default createBackendPlugin({
     let gitopsCollectorRegistry: CollectorRegistry | undefined;
     let gitopsQueueManager: QueueManager | undefined;
     let gitopsConfigSecrets: HealthCheckSecretsDeps | undefined;
+    let gitopsCatalogClient: InferClient<typeof CatalogApi> | undefined;
+    // Resolved AdvisoryLockService captured in init() for use in
+    // afterPluginsReady (the boot reconcile serializes across pods on it).
+    let resolvedAdvisoryLock: AdvisoryLockService | undefined;
+    // SignalService captured in init() for the afterPluginsReady rollup consumer.
+    let resolvedSignalService: SignalService | undefined;
     let healthCheckCache:
       | ReturnType<typeof createHealthCheckCache>
       | undefined;
@@ -211,6 +221,15 @@ export default createBackendPlugin({
           throw new Error("QueueManager not initialized");
         return gitopsQueueManager;
       },
+      getDb: () => {
+        if (!gitopsDb) throw new Error("Healthcheck database not initialized");
+        return gitopsDb;
+      },
+      getCatalogClient: () => {
+        if (!gitopsCatalogClient)
+          throw new Error("Catalog client not initialized");
+        return gitopsCatalogClient;
+      },
     });
 
     env.registerInit({
@@ -317,6 +336,9 @@ export default createBackendPlugin({
         gitopsHealthCheckRegistry = healthCheckRegistry;
         gitopsCollectorRegistry = collectorRegistry;
         gitopsQueueManager = queueManager;
+        gitopsCatalogClient = rpcClient.forPlugin(CatalogApi);
+        resolvedAdvisoryLock = advisoryLock;
+        resolvedSignalService = signalService;
 
         // Bind the COMPUTE-ON-READ accessor's db + service for the `health`
         // entity (defined in register()). From here onward the entity `read`
@@ -553,8 +575,8 @@ export default createBackendPlugin({
               return mapped;
             },
           },
-          recomputeSystemRollupHealth: (systemId) =>
-            recomputeSystemRollupHealth({
+          recomputeSystemRollupHealth: async (systemId) => {
+            await recomputeSystemRollupHealth({
               systemId,
               // Reuse the COMPUTE-ON-READ service instance bound to the
               // `health` entity read accessor — it's the same db/registry
@@ -563,7 +585,8 @@ export default createBackendPlugin({
               getHealthEntity: () => healthEntity,
               advisoryLock,
               logger,
-            }),
+            });
+          },
         });
         rpc.registerRouter(healthCheckRouter, healthCheckContract);
 
@@ -607,11 +630,17 @@ export default createBackendPlugin({
       }) => {
         // Store emitHook for the queue worker (Closure-based Hook Getter pattern)
         storedEmitHook = emitHook;
-        // Bootstrap all enabled health checks
-        await bootstrapHealthChecks({
+        // Converge the per-environment recurring job set at boot (schedule
+        // desired (config, system, env) jobs, cancel orphans incl. old-format
+        // ones). The periodic reconcile below keeps it converged as catalog
+        // membership changes.
+        const reconcileCatalogClient = rpcClient.forPlugin(CatalogApi);
+        await reconcileHealthCheckJobs({
           db: database,
           queueManager,
+          catalogClient: reconcileCatalogClient,
           logger,
+          advisoryLock: resolvedAdvisoryLock,
         });
 
         // Notification subscription specs. Per-resource group lifecycle
@@ -649,11 +678,34 @@ export default createBackendPlugin({
         for (const action of createHealthCheckActions({
           service,
           queueManager,
+          catalogClient: reconcileCatalogClient,
           emitHook,
         })) {
           automationActions.registerAction(action, pluginMetadata);
         }
 
+        // Phase 2: event-driven debounced system-rollup consumer. Under
+        // per-environment jobs each run writes only its own env entity; this
+        // subscribes to per-env `health` changes and debounces a recompute of
+        // the bare `<systemId>` rollup entity (+ SYSTEM_STATUS_CHANGED). Needs
+        // the cache/signal/lock resolved in init().
+        if (resolvedAdvisoryLock && resolvedSignalService && healthCheckCache) {
+          await setupRollupConsumer({
+            queueManager,
+            onEntityChanged: entityPoint.onEntityChanged,
+            service,
+            advisoryLock: resolvedAdvisoryLock,
+            signalService: resolvedSignalService,
+            cache: healthCheckCache,
+            getHealthEntity: () => healthEntity,
+            logger,
+          });
+        } else {
+          logger.warn(
+            "Health rollup consumer NOT wired: advisoryLock/signalService/cache unresolved after init",
+          );
+        }
+
         // React to catalog system deletion (tombstone) via the reactive
         // `catalog-system` entity instead of the (removed) `system.deleted`
         // hook (§10.4). `work-queue` delivery preserved: association cleanup

package/src/migration-chain-contract.test.ts

@@ -19,7 +19,10 @@ import { describe, expect, it } from "bun:test";
 import type { QueueManager } from "@checkstack/queue-api";
 import type { Hook } from "@checkstack/backend-api";
 import { stateThresholds } from "./state-thresholds-migrations";
-import { createHealthCheckActions } from "./automations";
+import {
+  createHealthCheckActions,
+  type HealthCheckActionDeps,
+} from "./automations";
 import type { HealthCheckService } from "./service";
 
 // `createHealthCheckActions` only constructs the action definitions; the deps
@@ -27,6 +30,8 @@ import type { HealthCheckService } from "./service";
 // Stubs are sufficient.
 const stubService = {} as unknown as HealthCheckService;
 const stubQueueManager = {} as unknown as QueueManager;
+const stubCatalogClient =
+  {} as unknown as HealthCheckActionDeps["catalogClient"];
 const stubEmitHook = async <T>(_hook: Hook<T>, _payload: T): Promise<void> => {};
 
 describe("healthcheck config migration-chain contract", () => {
@@ -42,6 +47,7 @@ describe("healthcheck config migration-chain contract", () => {
     const actions = createHealthCheckActions({
       service: stubService,
       queueManager: stubQueueManager,
+      catalogClient: stubCatalogClient,
       emitHook: stubEmitHook,
     });
     expect(actions.length).toBeGreaterThan(0);

package/src/notification-policy.test.ts

@@ -5,6 +5,7 @@ import type {
 } from "@checkstack/healthcheck-common";
 import {
   classifyTransition,
+  shouldEmitRollupNotification,
   shouldNotifyTransition,
   type TransitionKind,
 } from "./notification-policy";
@@ -79,6 +80,24 @@ describe("shouldNotifyTransition", () => {
   });
 });
 
+describe("shouldEmitRollupNotification", () => {
+  // Regression guard for the duplicate-notification bug: a fanned-out system
+  // whose environment goes unhealthy notified once per env ("... in env X").
+  // The rollup notification ("... is unhealthy") describes the same outage, so
+  // it must be suppressed whenever an environment already notified this tick.
+  it("suppresses the rollup notification when an environment already notified", () => {
+    expect(
+      shouldEmitRollupNotification({ anyEnvironmentNotified: true }),
+    ).toBe(false);
+  });
+
+  it("emits the rollup notification as a fallback when no environment notified", () => {
+    expect(
+      shouldEmitRollupNotification({ anyEnvironmentNotified: false }),
+    ).toBe(true);
+  });
+});
+
 describe("flapping scenario from the bug report", () => {
   // healthy → degraded → unhealthy → degraded → healthy
   //

package/src/notification-policy.ts

@@ -54,3 +54,29 @@ export function shouldNotifyTransition(
   if (kind === "deescalation" && policy.suppressDeEscalations) return false;
   return true;
 }
+
+/**
+ * Decide whether the system-ROLLUP notification should fire for a fanned-out
+ * system in a given tick.
+ *
+ * When a check fans out to environments, each environment that changes status
+ * emits its own notification ("system unhealthy in env X"). The post-loop
+ * rollup transition ("system unhealthy") then describes the SAME underlying
+ * outage, so firing it too produces the duplicate notification pair users see.
+ *
+ * The rollup change is always driven by the very environment(s) that already
+ * notified this tick, so the rollup notification is redundant whenever any
+ * environment notified. It is only emitted as a fallback when NO environment
+ * notified (e.g. every per-env delivery was suppressed or threw) so a real
+ * system status change never goes entirely unannounced.
+ *
+ * The rollup TRANSITION record and the `SYSTEM_STATUS_CHANGED` signal are
+ * emitted regardless — only the user-facing notification is deduplicated.
+ */
+export function shouldEmitRollupNotification({
+  anyEnvironmentNotified,
+}: {
+  anyEnvironmentNotified: boolean;
+}): boolean {
+  return !anyEnvironmentNotified;
+}