@checkstack/healthcheck-backend 1.3.0 → 1.5.0

package/src/index.ts

@@ -3,7 +3,6 @@ import {
   bootstrapHealthChecks,
 } from "./queue-executor";
 import { setupRetentionJob } from "./retention-job";
-import { setupAutoIncidentCloseJob } from "./auto-incident-close-job";
 import * as schema from "./schema";
 import {
   healthCheckAccessRules,
@@ -31,8 +30,19 @@ import {
   automationActionExtensionPoint,
   automationArtifactTypeExtensionPoint,
   automationTriggerExtensionPoint,
+  entityExtensionPoint,
+  type EntityHandle,
 } from "@checkstack/automation-backend";
+import {
+  HEALTH_ENTITY_KIND,
+  HealthEntityStateSchema,
+  createHealthEntityRead,
+  deriveHealthTriggerEvents,
+  healthChangeToPayload,
+  type HealthEntityState,
+} from "./health-entity";
 import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
+import { secretResolverRef } from "@checkstack/secrets-backend";
 import { createHealthCheckRouter } from "./router";
 import { HealthCheckService } from "./service";
 import {
@@ -41,11 +51,8 @@ import {
   healthCheckTriggers,
 } from "./automations";
 import { registerHealthcheckGitOpsKinds, registerHealthcheckGitOpsDocumentation } from "./healthcheck-gitops-kinds";
-import { catalogHooks } from "@checkstack/catalog-backend";
+import { CATALOG_SYSTEM_ENTITY_KIND } from "@checkstack/catalog-backend";
 import { satelliteHooks } from "@checkstack/satellite-backend";
-import { incidentHooks } from "@checkstack/incident-backend";
-import { eq, and, isNull } from "drizzle-orm";
-import { healthCheckAutoIncidents } from "./schema";
 import { CatalogApi } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import { IncidentApi } from "@checkstack/incident-common";
@@ -57,6 +64,21 @@ import { createHealthCheckCache } from "./cache";
 // Store emitHook reference for use during Phase 2 init
 let storedEmitHook: EmitHookFn | undefined;
 
+// The reactive `health` entity handle (§10.3). Defined in register() via
+// the entity extension point (buffered until automation-backend registers
+// the impl); mutations only fire from init() onward once the read accessor
+// has its db + service.
+let healthEntity: EntityHandle<HealthEntityState> | undefined;
+
+// PLUGIN-BACKED + COMPUTED kind: the `health` aggregate has no domain table and
+// no framework `entity_state` row — its current state is COMPUTED on read from
+// the durable `health_check_runs` (via `getSystemHealthStatus`). The db +
+// service are only available in init(), but the entity `read` accessor must be
+// supplied at `defineEntity` time in register(). These holders bridge the two;
+// init() sets them before any mutation runs (the queue worker — the only
+// mutation site — is set up in init() after these are bound).
+let healthEntityService: HealthCheckService | undefined;
+
 export default createBackendPlugin({
   metadata: pluginMetadata,
   register(env) {
@@ -80,6 +102,47 @@ export default createBackendPlugin({
       .getExtensionPoint(automationArtifactTypeExtensionPoint)
       .registerArtifactType(assignmentArtifactType, pluginMetadata);
 
+    // ─── Reactive `health` entity (§10.3) ──────────────────────────────
+    // PLUGIN-BACKED + COMPUTED kind (Model B): the per-system aggregate has no
+    // domain table and NO framework `entity_state` row. `read` COMPUTES each
+    // system's `{ status, healthyChecks, totalChecks }` on demand from the same
+    // durable `health_check_runs` the rest of the plugin reads (via
+    // `getSystemHealthStatus`), gated on the system having at least one ENABLED
+    // check association — see `createHealthEntityRead`. A system with an enabled
+    // check but no runs yet resolves to the default-`healthy` baseline so a
+    // first-ever unhealthy run is a real `healthy → degraded` diff. The service
+    // is resolved in init() and bridged via the holder. The change →
+    // trigger-event deriver keeps the
+    // existing `healthcheck.system.degraded` / `.healthy` / `.health_changed`
+    // automations firing off the computed state.
+    const entityPoint = env.getExtensionPoint(entityExtensionPoint);
+    healthEntity = entityPoint.defineEntity<HealthEntityState>({
+      kind: HEALTH_ENTITY_KIND,
+      state: HealthEntityStateSchema,
+      read: (ids) => {
+        const service = healthEntityService;
+        if (!service) {
+          throw new Error(
+            "health entity read before init: service not yet resolved",
+          );
+        }
+        return createHealthEntityRead({ service })(ids);
+      },
+    });
+    entityPoint.registerChangeDeriver({
+      kind: HEALTH_ENTITY_KIND,
+      derive: deriveHealthTriggerEvents,
+      toPayload: healthChangeToPayload,
+    });
+    // Raw per-check samples + cursors are intentionally NON-reactive (§5):
+    // a firehose of individual runs would melt the wake-index; the
+    // aggregate is the entity.
+    entityPoint.declareNonReactiveState({
+      table: "health_check_runs",
+      reason: "raw-sample",
+      note: "High-frequency individual check executions. The per-system aggregate is the `health` entity; raw runs stay a numeric_state wake source only.",
+    });
+
     // ─── GitOps Entity Kind Registration ───────────────────────────────
     // Mutable refs — populated during init(), consumed by reconcile closures.
     let gitopsDb: SafeDatabase<typeof schema> | undefined;
@@ -134,6 +197,8 @@ export default createBackendPlugin({
         signalService: coreServices.signalService,
         cacheManager: coreServices.cacheManager,
         config: coreServices.config,
+        secretResolver: secretResolverRef,
+        advisoryLock: coreServices.advisoryLock,
       },
       // Phase 2: Register router and setup worker
       init: async ({
@@ -147,6 +212,8 @@ export default createBackendPlugin({
         signalService,
         cacheManager,
         config,
+        secretResolver,
+        advisoryLock,
       }) => {
         logger.debug("🏥 Initializing Health Check Backend...");
 
@@ -156,6 +223,17 @@ export default createBackendPlugin({
         gitopsCollectorRegistry = collectorRegistry;
         gitopsQueueManager = queueManager;
 
+        // Bind the COMPUTE-ON-READ accessor's db + service for the `health`
+        // entity (defined in register()). From here onward the entity `read`
+        // computes each system's aggregate from durable `health_check_runs`,
+        // and the queue worker (set up just below — the only mutation site)
+        // drives writes through `handle.mutate`.
+        healthEntityService = new HealthCheckService(
+          database,
+          healthCheckRegistry,
+          collectorRegistry,
+        );
+
         // Create catalog client for notification delegation
         const catalogClient = rpcClient.forPlugin(CatalogApi);
 
@@ -182,6 +260,7 @@ export default createBackendPlugin({
         await setupHealthCheckWorker({
           notificationClient,
           db: database,
+          advisoryLock,
           registry: healthCheckRegistry,
           collectorRegistry,
           logger,
@@ -191,7 +270,9 @@ export default createBackendPlugin({
           maintenanceClient,
           incidentClient,
           getEmitHook: () => storedEmitHook,
+          getHealthEntity: () => healthEntity,
           cache,
+          secretResolver,
         });
 
         // Setup retention job for tiered storage (daily aggregation)
@@ -201,15 +282,12 @@ export default createBackendPlugin({
           queueManager,
         });
 
-        // Setup auto-incident close worker (ticks every 60s, closes
-        // auto-opened incidents whose systems have been steady-healthy
-        // for the cooldown).
-        await setupAutoIncidentCloseJob({
-          db: database,
-          logger,
-          queueManager,
-          incidentClient,
-        });
+        // The hardcoded auto-incident open/close path was removed — auto-
+        // incident behaviour is now built entirely by user automations
+        // (e.g. `healthcheck.system_degraded` + `for:` → `incident.create`).
+        // Flapping is detected by the automation engine's windowed-count gate
+        // on the `system_health_changed` trigger — healthcheck emits only the
+        // raw aggregated-health change (via the reactive `health` entity).
 
         const healthCheckRouter = createHealthCheckRouter({
           database: database as SafeDatabase<typeof schema>,
@@ -220,6 +298,8 @@ export default createBackendPlugin({
           cache,
           configService: config,
           catalogClient,
+          maintenanceClient,
+          logger,
         });
         rpc.registerRouter(healthCheckRouter, healthCheckContract);
 
@@ -310,17 +390,23 @@ export default createBackendPlugin({
           automationActions.registerAction(action, pluginMetadata);
         }
 
-        onHook(
-          catalogHooks.systemDeleted,
-          async (payload) => {
+        // React to catalog system deletion (tombstone) via the reactive
+        // `catalog-system` entity instead of the (removed) `system.deleted`
+        // hook (§10.4). `work-queue` delivery preserved: association cleanup
+        // must run once per cluster, not per-instance.
+        entityPoint.onEntityChanged({
+          kind: CATALOG_SYSTEM_ENTITY_KIND,
+          handler: async (change) => {
+            if (change.next !== null) return; // tombstone only
+            const systemId = change.id;
             logger.debug(
-              `Cleaning up health check associations for deleted system: ${payload.systemId}`,
+              `Cleaning up health check associations for deleted system: ${systemId}`,
             );
-            await service.removeAllSystemAssociations(payload.systemId);
-            await healthCheckCache?.invalidateSystem(payload.systemId);
+            await service.removeAllSystemAssociations(systemId);
+            await healthCheckCache?.invalidateSystem(systemId);
           },
-          { mode: "work-queue", workerGroup: "system-cleanup" },
-        );
+          delivery: { mode: "work-queue", workerGroup: "system-cleanup" },
+        });
 
         // Subscribe to satellite deletion to scrub satellite IDs from associations
         onHook(
@@ -337,32 +423,6 @@ export default createBackendPlugin({
           { mode: "work-queue", workerGroup: "satellite-cleanup" },
         );
 
-        // Sync our auto-incident mapping when an incident is resolved.
-        // Without this, a manually-closed incident would still appear
-        // "active" in our mapping, blocking the require-recovery rule
-        // from re-evaluating fresh transitions.
-        onHook(
-          incidentHooks.incidentResolved,
-          async ({ incidentId }) => {
-            const updated = await database
-              .update(healthCheckAutoIncidents)
-              .set({ closedAt: new Date() })
-              .where(
-                and(
-                  eq(healthCheckAutoIncidents.incidentId, incidentId),
-                  isNull(healthCheckAutoIncidents.closedAt),
-                ),
-              )
-              .returning({ id: healthCheckAutoIncidents.id });
-            if (updated.length > 0) {
-              logger.debug(
-                `Marked auto-incident mapping closed for resolved incident ${incidentId}`,
-              );
-            }
-          },
-          { mode: "work-queue", workerGroup: "auto-incident-sync" },
-        );
-
         logger.debug("✅ Health Check Backend afterPluginsReady complete.");
       },
     });
@@ -371,3 +431,13 @@ export default createBackendPlugin({
 
 // Re-export hooks for other plugins to use
 export { healthCheckHooks } from "./hooks";
+
+// Re-export the reactive `health` entity surface so cross-plugin consumers
+// (slo, dependency) can subscribe via onEntityChanged + classify changes
+// without duplicating the kind id / transition predicate (§10.3).
+export {
+  HEALTH_ENTITY_KIND,
+  classifyHealthChange,
+  type HealthChangeClassification,
+  type HealthEntityState,
+} from "./health-entity";

package/src/queue-executor.test.ts

@@ -13,6 +13,16 @@ const passthroughCache: HealthCheckCache = {
   invalidateAllSystems: async () => 0,
   scope: {} as HealthCheckCache["scope"],
 };
+
+// Pass-through advisory lock: these tests don't exercise cross-pod
+// serialization, so run the critical section directly.
+const mockAdvisoryLock: Parameters<
+  typeof setupHealthCheckWorker
+>[0]["advisoryLock"] = {
+  tryAcquire: async () => ({ release: async () => {} }),
+  withXactLock: <T>({ fn }: { key: string; fn: () => Promise<T> }): Promise<T> =>
+    fn(),
+};
 import {
   createMockLogger,
   createMockQueueManager,
@@ -179,6 +189,7 @@ describe("Queue-Based Health Check Executor", () => {
         db: mockDb as unknown as Parameters<
           typeof setupHealthCheckWorker
         >[0]["db"],
+        advisoryLock: mockAdvisoryLock,
         registry: mockRegistry,
         collectorRegistry:
           createMockCollectorRegistry() as unknown as Parameters<
@@ -376,6 +387,7 @@ describe("Queue-Based Health Check Executor", () => {
         db: mockDb as unknown as Parameters<
           typeof setupHealthCheckWorker
         >[0]["db"],
+        advisoryLock: mockAdvisoryLock,
         registry: mockRegistry,
         collectorRegistry:
           createMockCollectorRegistry() as unknown as Parameters<
@@ -510,6 +522,7 @@ describe("Queue-Based Health Check Executor", () => {
         db: mockDb as unknown as Parameters<
           typeof setupHealthCheckWorker
         >[0]["db"],
+        advisoryLock: mockAdvisoryLock,
         registry: mockRegistry,
         collectorRegistry: mockCollectorRegistry as unknown as Parameters<
           typeof setupHealthCheckWorker