@checkstack/healthcheck-backend 1.16.0 → 1.18.0

package/src/automations.test.ts

@@ -10,6 +10,7 @@ import {
   assignmentArtifactType,
   checkFailedTrigger,
   createHealthCheckActions,
+  type HealthCheckActionDeps,
   healthCheckTriggers,
   systemDegradedTrigger,
   systemHealthChangedTrigger,
@@ -146,13 +147,18 @@ describe("assignmentArtifactType", () => {
 
 function makeService(args: {
   setAssignmentEnabledReturn?: boolean;
+  enqueueEnvironmentIds?: (string | null)[];
 }): HealthCheckService & { setMock: ReturnType<typeof mock> } {
   const setMock = mock(
     async (_sysId: string, _cfgId: string, _enabled: boolean) =>
       args.setAssignmentEnabledReturn ?? true,
   );
+  const resolveEnqueueEnvironmentIds = mock(
+    async () => args.enqueueEnvironmentIds ?? [null],
+  );
   return {
     setAssignmentEnabled: setMock,
+    resolveEnqueueEnvironmentIds,
     setMock,
   } as unknown as HealthCheckService & { setMock: ReturnType<typeof mock> };
 }
@@ -174,6 +180,11 @@ function makeQueueManager(): QueueEnqueueRecorder {
   return { queueManager, enqueueMock };
 }
 
+// The actions only need a catalog client shape for `run_now`, which delegates
+// environment resolution to the service mock, so a bare stub suffices.
+const catalogClientStub =
+  {} as unknown as HealthCheckActionDeps["catalogClient"];
+
 describe("healthcheck.run_now", () => {
   it("enqueues a one-off job and emits an enqueued=true artifact", async () => {
     const service = makeService({});
@@ -182,6 +193,7 @@ describe("healthcheck.run_now", () => {
     const [runNow] = createHealthCheckActions({
       service,
       queueManager,
+      catalogClient: catalogClientStub,
       emitHook: emitHook as never,
     });
 
@@ -198,10 +210,42 @@ describe("healthcheck.run_now", () => {
     expect(enqueueMock.mock.calls[0]![0]).toEqual({
       configId: "cfg-1",
       systemId: "sys-1",
+      environmentId: null,
     });
     // run_now doesn't mutate any DB row → no hook to emit.
     expect(emitHook).not.toHaveBeenCalled();
   });
+
+  it("enqueues one job per effective environment slice", async () => {
+    const service = makeService({ enqueueEnvironmentIds: ["prod", "staging"] });
+    const { queueManager, enqueueMock } = makeQueueManager();
+    const emitHook = mock(async (_hook: unknown, _payload: unknown) => {});
+    const [runNow] = createHealthCheckActions({
+      service,
+      queueManager,
+      catalogClient: catalogClientStub,
+      emitHook: emitHook as never,
+    });
+
+    const result = await runNow!.execute({
+      ...ctxBase,
+      consumedArtifacts: {},
+      config: { systemId: "sys-1", configurationId: "cfg-1" } as never,
+    });
+
+    expect(result.success).toBe(true);
+    expect(enqueueMock).toHaveBeenCalledTimes(2);
+    expect(enqueueMock.mock.calls[0]![0]).toEqual({
+      configId: "cfg-1",
+      systemId: "sys-1",
+      environmentId: "prod",
+    });
+    expect(enqueueMock.mock.calls[1]![0]).toEqual({
+      configId: "cfg-1",
+      systemId: "sys-1",
+      environmentId: "staging",
+    });
+  });
 });
 
 describe("healthcheck.enable_assignment", () => {
@@ -212,6 +256,7 @@ describe("healthcheck.enable_assignment", () => {
     const [, enable] = createHealthCheckActions({
       service,
       queueManager,
+      catalogClient: catalogClientStub,
       emitHook: emitHook as never,
     });
 
@@ -236,6 +281,7 @@ describe("healthcheck.enable_assignment", () => {
     const [, enable] = createHealthCheckActions({
       service,
       queueManager,
+      catalogClient: catalogClientStub,
       emitHook: emitHook as never,
     });
 
@@ -260,6 +306,7 @@ describe("healthcheck.disable_assignment", () => {
     const [, , disable] = createHealthCheckActions({
       service,
       queueManager,
+      catalogClient: catalogClientStub,
       emitHook: emitHook as never,
     });
 

package/src/automations.ts

@@ -26,6 +26,8 @@
 import { z } from "zod";
 import { Versioned, type Hook } from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
+import type { InferClient } from "@checkstack/common";
+import type { CatalogApi } from "@checkstack/catalog-common";
 import type {
   ActionDefinition,
   TriggerDefinition,
@@ -234,6 +236,7 @@ export const assignmentArtifactType = {
 export interface HealthCheckActionDeps {
   service: HealthCheckService;
   queueManager: QueueManager;
+  catalogClient: InferClient<typeof CatalogApi>;
   emitHook: <T>(hook: Hook<T>, payload: T) => Promise<void>;
 }
 
@@ -256,12 +259,25 @@ export function createHealthCheckActions(
       const queue = deps.queueManager.getQueue<HealthCheckJobPayload>(
         HEALTH_CHECK_QUEUE,
       );
-      await queue.enqueue({
-        configId: config.configurationId,
+      // Enqueue one one-off job per effective environment slice so a manual
+      // run covers exactly the same slices the recurring schedule does. An
+      // assignment with no effective environments enqueues a single env-less
+      // run (`environmentId: null`).
+      const environmentIds = await deps.service.resolveEnqueueEnvironmentIds({
         systemId: config.systemId,
+        configurationId: config.configurationId,
+        catalogClient: deps.catalogClient,
+        logger,
       });
+      for (const environmentId of environmentIds) {
+        await queue.enqueue({
+          configId: config.configurationId,
+          systemId: config.systemId,
+          environmentId,
+        });
+      }
       logger.info(
-        `Automation enqueued run for ${config.systemId}:${config.configurationId}`,
+        `Automation enqueued ${environmentIds.length} run(s) for ${config.systemId}:${config.configurationId}`,
       );
       return {
         success: true,

package/src/healthcheck-gitops-kinds.test.ts

@@ -20,6 +20,34 @@ import { Versioned } from "@checkstack/backend-api";
  * System → healthchecks extension in isolation with mock services.
  */
 
+// The System extension's reconcile converges the per-env recurring jobs via
+// `reconcileHealthCheckJobs`. That path has its own dedicated tests
+// (schedule-reconciler.test.ts); here we feed it EMPTY db/catalog/queue stubs so
+// it is a safe no-op and these tests can focus on the association logic.
+// (A `mock.module` would leak process-wide and break schedule-reconciler.test.)
+function emptyReconcileDbStub() {
+  return {
+    select: () => ({
+      from: () => ({
+        innerJoin: () => ({ where: () => Promise.resolve([]) }),
+        groupBy: () => Promise.resolve([]),
+      }),
+    }),
+  } as never;
+}
+function emptyReconcileQueueManager() {
+  return {
+    getQueue: () => ({
+      scheduleRecurring: async () => "job-123",
+      listRecurringJobs: async () => [],
+      getRecurringJobDetails: async () => undefined,
+      cancelRecurring: async () => {},
+    }),
+  } as never;
+}
+const emptyCatalogClient = () =>
+  ({ resolveSystemEnvironments: async () => [] }) as never;
+
 // ─── Mock Healthcheck Service ──────────────────────────────────────────────
 
 interface MockConfig {
@@ -231,7 +259,9 @@ describe("Healthcheck GitOps Kind: Healthcheck", () => {
       createService: () => mockService as any,
       getHealthCheckRegistry: () => mockHCRegistry as any,
       getCollectorRegistry: () => mockCollectorRegistry as any,
-      getQueueManager: () => ({ getQueue: () => ({ scheduleRecurring: async () => "job-123" }) } as any),
+      getQueueManager: () => emptyReconcileQueueManager(),
+      getDb: () => emptyReconcileDbStub(),
+      getCatalogClient: () => emptyCatalogClient(),
     };
     return buildHealthcheckKind(mockDeps);
   }
@@ -594,7 +624,9 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
         }) as never,
       getHealthCheckRegistry: () => createMockHealthCheckRegistry() as never,
       getCollectorRegistry: () => createMockCollectorRegistry() as never,
-      getQueueManager: () => ({ getQueue: () => ({ scheduleRecurring: async () => "job-123" }) } as any),
+      getQueueManager: () => emptyReconcileQueueManager(),
+      getDb: () => emptyReconcileDbStub(),
+      getCatalogClient: () => emptyCatalogClient(),
     });
   }
 

package/src/healthcheck-gitops-kinds.ts

@@ -25,7 +25,11 @@ import {
   enumField,
 } from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
-import { scheduleHealthCheck } from "./queue-executor";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import type { InferClient } from "@checkstack/common";
+import type { CatalogApi } from "@checkstack/catalog-common";
+import type * as schema from "./schema";
+import { reconcileHealthCheckJobs } from "./schedule-reconciler";
 
 /**
  * Lazy accessor functions — populated during init(), consumed during reconcile.
@@ -37,6 +41,8 @@ interface HealthcheckGitOpsKindsDeps {
   getHealthCheckRegistry: () => HealthCheckRegistry;
   getCollectorRegistry: () => CollectorRegistry;
   getQueueManager: () => QueueManager;
+  getDb: () => SafeDatabase<typeof schema>;
+  getCatalogClient: () => InferClient<typeof CatalogApi>;
 }
 
 // ─── Healthcheck Spec Schema ───────────────────────────────────────────────
@@ -382,18 +388,16 @@ export function buildSystemHealthcheckExtension(
           notificationPolicy,
         });
 
-        // Retrieve config to get the interval for scheduling
-        const config = await service.getConfiguration(configId);
-        if (config) {
-          await scheduleHealthCheck({
-            queueManager: deps.getQueueManager(),
-            payload: {
-              configId,
-              systemId: systemEntityId,
-            },
-            intervalSeconds: config.intervalSeconds,
-          });
-        }
+        // Reconcile this system's per-env recurring jobs so the new
+        // association starts probing right away (system-scoped: add/update
+        // only, orphan cleanup is owned by the periodic full reconcile).
+        await reconcileHealthCheckJobs({
+          db: deps.getDb(),
+          queueManager: deps.getQueueManager(),
+          catalogClient: deps.getCatalogClient(),
+          logger: context.logger,
+          systemId: systemEntityId,
+        });
 
         context.logger.info(
           `GitOps: associated ${entry.ref.kind} "${entry.ref.name}" (${configId}) with System "${entity.metadata.name}" and scheduled execution`,

package/src/index.ts

@@ -1,8 +1,8 @@
 import {
   setupHealthCheckWorker,
-  bootstrapHealthChecks,
   recomputeSystemRollupHealth,
 } from "./queue-executor";
+import { reconcileHealthCheckJobs } from "./schedule-reconciler";
 import { setupRetentionJob } from "./retention-job";
 import * as schema from "./schema";
 import {
@@ -38,6 +38,7 @@ import {
   type SafeDatabase,
   type HealthCheckRegistry,
   type CollectorRegistry,
+  type AdvisoryLockService,
 } from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
 import {
@@ -75,6 +76,9 @@ import { IncidentApi } from "@checkstack/incident-common";
 import { GitOpsApi } from "@checkstack/gitops-common";
 import { registerSearchProvider } from "@checkstack/command-backend";
 import { resolveRoute } from "@checkstack/common";
+import type { InferClient } from "@checkstack/common";
+import type { SignalService } from "@checkstack/signal-common";
+import { setupRollupConsumer } from "./rollup-consumer";
 import { createHealthCheckCache } from "./cache";
 import { inArray, ilike } from "drizzle-orm";
 
@@ -174,6 +178,12 @@ export default createBackendPlugin({
     let gitopsCollectorRegistry: CollectorRegistry | undefined;
     let gitopsQueueManager: QueueManager | undefined;
     let gitopsConfigSecrets: HealthCheckSecretsDeps | undefined;
+    let gitopsCatalogClient: InferClient<typeof CatalogApi> | undefined;
+    // Resolved AdvisoryLockService captured in init() for use in
+    // afterPluginsReady (the boot reconcile serializes across pods on it).
+    let resolvedAdvisoryLock: AdvisoryLockService | undefined;
+    // SignalService captured in init() for the afterPluginsReady rollup consumer.
+    let resolvedSignalService: SignalService | undefined;
     let healthCheckCache:
       | ReturnType<typeof createHealthCheckCache>
       | undefined;
@@ -211,6 +221,15 @@ export default createBackendPlugin({
           throw new Error("QueueManager not initialized");
         return gitopsQueueManager;
       },
+      getDb: () => {
+        if (!gitopsDb) throw new Error("Healthcheck database not initialized");
+        return gitopsDb;
+      },
+      getCatalogClient: () => {
+        if (!gitopsCatalogClient)
+          throw new Error("Catalog client not initialized");
+        return gitopsCatalogClient;
+      },
     });
 
     env.registerInit({
@@ -317,6 +336,9 @@ export default createBackendPlugin({
         gitopsHealthCheckRegistry = healthCheckRegistry;
         gitopsCollectorRegistry = collectorRegistry;
         gitopsQueueManager = queueManager;
+        gitopsCatalogClient = rpcClient.forPlugin(CatalogApi);
+        resolvedAdvisoryLock = advisoryLock;
+        resolvedSignalService = signalService;
 
         // Bind the COMPUTE-ON-READ accessor's db + service for the `health`
         // entity (defined in register()). From here onward the entity `read`
@@ -524,8 +546,37 @@ export default createBackendPlugin({
           logger,
           signalService,
           configSecrets,
-          recomputeSystemRollupHealth: (systemId) =>
-            recomputeSystemRollupHealth({
+          // Fold active incident health overrides into the user-facing system
+          // health reads (worst-wins). Reuses the incident client already built
+          // above; maps the incident rows to the source-agnostic override shape
+          // the fold expects. Kept out of the shared deriver so SLO/AI/entity
+          // paths stay checks-only (see router.ts).
+          incidentHealthOverrideReader: {
+            getActiveOverrides: async (systemIds) => {
+              const { overrides } =
+                await incidentClient.getActiveHealthOverrides({ systemIds });
+              const mapped: Record<
+                string,
+                {
+                  status: "degraded" | "unhealthy";
+                  source: string;
+                  reason: string;
+                  sourceId?: string;
+                }[]
+              > = {};
+              for (const [systemId, list] of Object.entries(overrides)) {
+                mapped[systemId] = list.map((o) => ({
+                  status: o.status,
+                  source: "incident",
+                  reason: o.incidentTitle,
+                  sourceId: o.incidentId,
+                }));
+              }
+              return mapped;
+            },
+          },
+          recomputeSystemRollupHealth: async (systemId) => {
+            await recomputeSystemRollupHealth({
               systemId,
               // Reuse the COMPUTE-ON-READ service instance bound to the
               // `health` entity read accessor — it's the same db/registry
@@ -534,7 +585,8 @@ export default createBackendPlugin({
               getHealthEntity: () => healthEntity,
               advisoryLock,
               logger,
-            }),
+            });
+          },
         });
         rpc.registerRouter(healthCheckRouter, healthCheckContract);
 
@@ -578,11 +630,17 @@ export default createBackendPlugin({
       }) => {
         // Store emitHook for the queue worker (Closure-based Hook Getter pattern)
         storedEmitHook = emitHook;
-        // Bootstrap all enabled health checks
-        await bootstrapHealthChecks({
+        // Converge the per-environment recurring job set at boot (schedule
+        // desired (config, system, env) jobs, cancel orphans incl. old-format
+        // ones). The periodic reconcile below keeps it converged as catalog
+        // membership changes.
+        const reconcileCatalogClient = rpcClient.forPlugin(CatalogApi);
+        await reconcileHealthCheckJobs({
           db: database,
           queueManager,
+          catalogClient: reconcileCatalogClient,
           logger,
+          advisoryLock: resolvedAdvisoryLock,
         });
 
         // Notification subscription specs. Per-resource group lifecycle
@@ -620,11 +678,34 @@ export default createBackendPlugin({
         for (const action of createHealthCheckActions({
           service,
           queueManager,
+          catalogClient: reconcileCatalogClient,
           emitHook,
         })) {
           automationActions.registerAction(action, pluginMetadata);
         }
 
+        // Phase 2: event-driven debounced system-rollup consumer. Under
+        // per-environment jobs each run writes only its own env entity; this
+        // subscribes to per-env `health` changes and debounces a recompute of
+        // the bare `<systemId>` rollup entity (+ SYSTEM_STATUS_CHANGED). Needs
+        // the cache/signal/lock resolved in init().
+        if (resolvedAdvisoryLock && resolvedSignalService && healthCheckCache) {
+          await setupRollupConsumer({
+            queueManager,
+            onEntityChanged: entityPoint.onEntityChanged,
+            service,
+            advisoryLock: resolvedAdvisoryLock,
+            signalService: resolvedSignalService,
+            cache: healthCheckCache,
+            getHealthEntity: () => healthEntity,
+            logger,
+          });
+        } else {
+          logger.warn(
+            "Health rollup consumer NOT wired: advisoryLock/signalService/cache unresolved after init",
+          );
+        }
+
         // React to catalog system deletion (tombstone) via the reactive
         // `catalog-system` entity instead of the (removed) `system.deleted`
         // hook (§10.4). `work-queue` delivery preserved: association cleanup

package/src/migration-chain-contract.test.ts

@@ -19,7 +19,10 @@ import { describe, expect, it } from "bun:test";
 import type { QueueManager } from "@checkstack/queue-api";
 import type { Hook } from "@checkstack/backend-api";
 import { stateThresholds } from "./state-thresholds-migrations";
-import { createHealthCheckActions } from "./automations";
+import {
+  createHealthCheckActions,
+  type HealthCheckActionDeps,
+} from "./automations";
 import type { HealthCheckService } from "./service";
 
 // `createHealthCheckActions` only constructs the action definitions; the deps
@@ -27,6 +30,8 @@ import type { HealthCheckService } from "./service";
 // Stubs are sufficient.
 const stubService = {} as unknown as HealthCheckService;
 const stubQueueManager = {} as unknown as QueueManager;
+const stubCatalogClient =
+  {} as unknown as HealthCheckActionDeps["catalogClient"];
 const stubEmitHook = async <T>(_hook: Hook<T>, _payload: T): Promise<void> => {};
 
 describe("healthcheck config migration-chain contract", () => {
@@ -42,6 +47,7 @@ describe("healthcheck config migration-chain contract", () => {
     const actions = createHealthCheckActions({
       service: stubService,
       queueManager: stubQueueManager,
+      catalogClient: stubCatalogClient,
       emitHook: stubEmitHook,
     });
     expect(actions.length).toBeGreaterThan(0);

package/src/notification-policy.test.ts

@@ -5,6 +5,7 @@ import type {
 } from "@checkstack/healthcheck-common";
 import {
   classifyTransition,
+  shouldEmitRollupNotification,
   shouldNotifyTransition,
   type TransitionKind,
 } from "./notification-policy";
@@ -79,6 +80,24 @@ describe("shouldNotifyTransition", () => {
   });
 });
 
+describe("shouldEmitRollupNotification", () => {
+  // Regression guard for the duplicate-notification bug: a fanned-out system
+  // whose environment goes unhealthy notified once per env ("... in env X").
+  // The rollup notification ("... is unhealthy") describes the same outage, so
+  // it must be suppressed whenever an environment already notified this tick.
+  it("suppresses the rollup notification when an environment already notified", () => {
+    expect(
+      shouldEmitRollupNotification({ anyEnvironmentNotified: true }),
+    ).toBe(false);
+  });
+
+  it("emits the rollup notification as a fallback when no environment notified", () => {
+    expect(
+      shouldEmitRollupNotification({ anyEnvironmentNotified: false }),
+    ).toBe(true);
+  });
+});
+
 describe("flapping scenario from the bug report", () => {
   // healthy → degraded → unhealthy → degraded → healthy
   //

package/src/notification-policy.ts

@@ -54,3 +54,29 @@ export function shouldNotifyTransition(
   if (kind === "deescalation" && policy.suppressDeEscalations) return false;
   return true;
 }
+
+/**
+ * Decide whether the system-ROLLUP notification should fire for a fanned-out
+ * system in a given tick.
+ *
+ * When a check fans out to environments, each environment that changes status
+ * emits its own notification ("system unhealthy in env X"). The post-loop
+ * rollup transition ("system unhealthy") then describes the SAME underlying
+ * outage, so firing it too produces the duplicate notification pair users see.
+ *
+ * The rollup change is always driven by the very environment(s) that already
+ * notified this tick, so the rollup notification is redundant whenever any
+ * environment notified. It is only emitted as a fallback when NO environment
+ * notified (e.g. every per-env delivery was suppressed or threw) so a real
+ * system status change never goes entirely unannounced.
+ *
+ * The rollup TRANSITION record and the `SYSTEM_STATUS_CHANGED` signal are
+ * emitted regardless — only the user-facing notification is deduplicated.
+ */
+export function shouldEmitRollupNotification({
+  anyEnvironmentNotified,
+}: {
+  anyEnvironmentNotified: boolean;
+}): boolean {
+  return !anyEnvironmentNotified;
+}