@checkstack/healthcheck-backend 1.1.4 → 1.3.0

package/src/index.ts

@@ -3,6 +3,7 @@ import {
   bootstrapHealthChecks,
 } from "./queue-executor";
 import { setupRetentionJob } from "./retention-job";
+import { setupAutoIncidentCloseJob } from "./auto-incident-close-job";
 import * as schema from "./schema";
 import {
   healthCheckAccessRules,
@@ -26,46 +27,33 @@ import {
   type CollectorRegistry,
 } from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
-import { integrationEventExtensionPoint } from "@checkstack/integration-backend";
+import {
+  automationActionExtensionPoint,
+  automationArtifactTypeExtensionPoint,
+  automationTriggerExtensionPoint,
+} from "@checkstack/automation-backend";
 import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
-import { z } from "zod";
 import { createHealthCheckRouter } from "./router";
 import { HealthCheckService } from "./service";
+import {
+  assignmentArtifactType,
+  createHealthCheckActions,
+  healthCheckTriggers,
+} from "./automations";
 import { registerHealthcheckGitOpsKinds, registerHealthcheckGitOpsDocumentation } from "./healthcheck-gitops-kinds";
 import { catalogHooks } from "@checkstack/catalog-backend";
 import { satelliteHooks } from "@checkstack/satellite-backend";
+import { incidentHooks } from "@checkstack/incident-backend";
+import { eq, and, isNull } from "drizzle-orm";
+import { healthCheckAutoIncidents } from "./schema";
 import { CatalogApi } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import { IncidentApi } from "@checkstack/incident-common";
 import { GitOpsApi } from "@checkstack/gitops-common";
-import { healthCheckHooks } from "./hooks";
 import { registerSearchProvider } from "@checkstack/command-backend";
 import { resolveRoute } from "@checkstack/common";
 import { createHealthCheckCache } from "./cache";
 
-// =============================================================================
-// Integration Event Payload Schemas
-// =============================================================================
-
-const systemDegradedPayloadSchema = z.object({
-  systemId: z.string(),
-  systemName: z.string().optional(),
-  previousStatus: z.string(),
-  newStatus: z.string(),
-  healthyChecks: z.number(),
-  totalChecks: z.number(),
-  timestamp: z.string(),
-});
-
-const systemHealthyPayloadSchema = z.object({
-  systemId: z.string(),
-  systemName: z.string().optional(),
-  previousStatus: z.string(),
-  healthyChecks: z.number(),
-  totalChecks: z.number(),
-  timestamp: z.string(),
-});
-
 // Store emitHook reference for use during Phase 2 init
 let storedEmitHook: EmitHookFn | undefined;
 
@@ -78,33 +66,19 @@ export default createBackendPlugin({
       healthcheckGroupSubscription,
     ]);
 
-    // Register hooks as integration events
-    const integrationEvents = env.getExtensionPoint(
-      integrationEventExtensionPoint,
-    );
-
-    integrationEvents.registerEvent(
-      {
-        hook: healthCheckHooks.systemDegraded,
-        displayName: "System Health Degraded",
-        description:
-          "Fired when a system's health status transitions from healthy to degraded/unhealthy",
-        category: "Health",
-        payloadSchema: systemDegradedPayloadSchema,
-      },
-      pluginMetadata,
-    );
-
-    integrationEvents.registerEvent(
-      {
-        hook: healthCheckHooks.systemHealthy,
-        displayName: "System Health Restored",
-        description: "Fired when a system's health status recovers to healthy",
-        category: "Health",
-        payloadSchema: systemHealthyPayloadSchema,
-      },
-      pluginMetadata,
+    // ─── Automation Platform: triggers + artifact type ─────────────────
+    // Buffered behind the extension point until automation-backend's
+    // register() runs. Actions are wired in afterPluginsReady where
+    // `emitHook` becomes available.
+    const automationTriggers = env.getExtensionPoint(
+      automationTriggerExtensionPoint,
     );
+    for (const trigger of healthCheckTriggers) {
+      automationTriggers.registerTrigger(trigger, pluginMetadata);
+    }
+    env
+      .getExtensionPoint(automationArtifactTypeExtensionPoint)
+      .registerArtifactType(assignmentArtifactType, pluginMetadata);
 
     // ─── GitOps Entity Kind Registration ───────────────────────────────
     // Mutable refs — populated during init(), consumed by reconcile closures.
@@ -159,6 +133,7 @@ export default createBackendPlugin({
         queueManager: coreServices.queueManager,
         signalService: coreServices.signalService,
         cacheManager: coreServices.cacheManager,
+        config: coreServices.config,
       },
       // Phase 2: Register router and setup worker
       init: async ({
@@ -171,6 +146,7 @@ export default createBackendPlugin({
         queueManager,
         signalService,
         cacheManager,
+        config,
       }) => {
         logger.debug("🏥 Initializing Health Check Backend...");
 
@@ -225,6 +201,16 @@ export default createBackendPlugin({
           queueManager,
         });
 
+        // Setup auto-incident close worker (ticks every 60s, closes
+        // auto-opened incidents whose systems have been steady-healthy
+        // for the cooldown).
+        await setupAutoIncidentCloseJob({
+          db: database,
+          logger,
+          queueManager,
+          incidentClient,
+        });
+
         const healthCheckRouter = createHealthCheckRouter({
           database: database as SafeDatabase<typeof schema>,
           registry: healthCheckRegistry,
@@ -232,6 +218,8 @@ export default createBackendPlugin({
           gitOpsClient,
           getEmitHook: () => storedEmitHook,
           cache,
+          configService: config,
+          catalogClient,
         });
         rpc.registerRouter(healthCheckRouter, healthCheckContract);
 
@@ -308,6 +296,20 @@ export default createBackendPlugin({
           healthCheckRegistry,
           collectorRegistry,
         );
+
+        // Register automation actions now that `emitHook` + `queueManager`
+        // are both available.
+        const automationActions = env.getExtensionPoint(
+          automationActionExtensionPoint,
+        );
+        for (const action of createHealthCheckActions({
+          service,
+          queueManager,
+          emitHook,
+        })) {
+          automationActions.registerAction(action, pluginMetadata);
+        }
+
         onHook(
           catalogHooks.systemDeleted,
           async (payload) => {
@@ -335,6 +337,32 @@ export default createBackendPlugin({
           { mode: "work-queue", workerGroup: "satellite-cleanup" },
         );
 
+        // Sync our auto-incident mapping when an incident is resolved.
+        // Without this, a manually-closed incident would still appear
+        // "active" in our mapping, blocking the require-recovery rule
+        // from re-evaluating fresh transitions.
+        onHook(
+          incidentHooks.incidentResolved,
+          async ({ incidentId }) => {
+            const updated = await database
+              .update(healthCheckAutoIncidents)
+              .set({ closedAt: new Date() })
+              .where(
+                and(
+                  eq(healthCheckAutoIncidents.incidentId, incidentId),
+                  isNull(healthCheckAutoIncidents.closedAt),
+                ),
+              )
+              .returning({ id: healthCheckAutoIncidents.id });
+            if (updated.length > 0) {
+              logger.debug(
+                `Marked auto-incident mapping closed for resolved incident ${incidentId}`,
+              );
+            }
+          },
+          { mode: "work-queue", workerGroup: "auto-incident-sync" },
+        );
+
         logger.debug("✅ Health Check Backend afterPluginsReady complete.");
       },
     });

package/src/notification-defaults-config.ts

@@ -0,0 +1,10 @@
+/**
+ * Versioned schema used by `ConfigService` to persist platform-wide
+ * notification defaults. The shape is the runtime `NotificationPolicy`
+ * itself — each field has a built-in compile-time default so an empty
+ * stored value still parses to a valid policy.
+ */
+export { NotificationPolicySchema as notificationDefaultsConfigV1 } from "@checkstack/healthcheck-common";
+
+export const NOTIFICATION_DEFAULTS_CONFIG_ID = "healthcheck.notification-defaults";
+export const NOTIFICATION_DEFAULTS_CONFIG_VERSION = 1;

package/src/notification-policy.test.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from "bun:test";
+import type {
+  HealthCheckStatus,
+  NotificationPolicy,
+} from "@checkstack/healthcheck-common";
+import {
+  classifyTransition,
+  shouldNotifyTransition,
+  type TransitionKind,
+} from "./notification-policy";
+
+const STATES: HealthCheckStatus[] = ["healthy", "degraded", "unhealthy"];
+
+describe("classifyTransition", () => {
+  // Build the full 3×3 transition matrix so future severity edits stay
+  // honest. Every cell here doubles as documentation.
+  const matrix: Record<
+    HealthCheckStatus,
+    Record<HealthCheckStatus, TransitionKind>
+  > = {
+    healthy: {
+      healthy: "none",
+      degraded: "escalation",
+      unhealthy: "escalation",
+    },
+    degraded: {
+      healthy: "recovery",
+      degraded: "none",
+      unhealthy: "escalation",
+    },
+    unhealthy: {
+      healthy: "recovery",
+      degraded: "deescalation",
+      unhealthy: "none",
+    },
+  };
+
+  for (const prev of STATES) {
+    for (const next of STATES) {
+      it(`${prev} → ${next} = ${matrix[prev][next]}`, () => {
+        expect(classifyTransition(prev, next)).toBe(matrix[prev][next]);
+      });
+    }
+  }
+});
+
+describe("shouldNotifyTransition", () => {
+  // The helper only reads `suppressDeEscalations`; narrow the fixture
+  // type so the test doesn't need to keep up with unrelated policy
+  // fields added over time.
+  const off: Pick<NotificationPolicy, "suppressDeEscalations"> = {
+    suppressDeEscalations: false,
+  };
+  const on: Pick<NotificationPolicy, "suppressDeEscalations"> = {
+    suppressDeEscalations: true,
+  };
+
+  it("never notifies on `none` (no actual change)", () => {
+    expect(shouldNotifyTransition("none", off)).toBe(false);
+    expect(shouldNotifyTransition("none", on)).toBe(false);
+  });
+
+  it("always notifies on escalations regardless of policy", () => {
+    expect(shouldNotifyTransition("escalation", off)).toBe(true);
+    expect(shouldNotifyTransition("escalation", on)).toBe(true);
+  });
+
+  it("always notifies on recoveries regardless of policy", () => {
+    expect(shouldNotifyTransition("recovery", off)).toBe(true);
+    expect(shouldNotifyTransition("recovery", on)).toBe(true);
+  });
+
+  it("notifies on de-escalations by default", () => {
+    expect(shouldNotifyTransition("deescalation", off)).toBe(true);
+  });
+
+  it("suppresses de-escalations when the policy opts in", () => {
+    expect(shouldNotifyTransition("deescalation", on)).toBe(false);
+  });
+});
+
+describe("flapping scenario from the bug report", () => {
+  // healthy → degraded → unhealthy → degraded → healthy
+  //
+  // With suppression on, the intermediate `unhealthy → degraded`
+  // notification (the one operators called out as spammy) must be
+  // skipped, while escalation and recovery still fire.
+  const policy: Pick<NotificationPolicy, "suppressDeEscalations"> = {
+    suppressDeEscalations: true,
+  };
+  const sequence: [HealthCheckStatus, HealthCheckStatus, boolean][] = [
+    ["healthy", "degraded", true], // escalation
+    ["degraded", "unhealthy", true], // escalation
+    ["unhealthy", "degraded", false], // de-escalation — suppressed
+    ["degraded", "healthy", true], // recovery
+  ];
+
+  for (const [prev, next, expected] of sequence) {
+    it(`${prev} → ${next} should notify: ${expected}`, () => {
+      const kind = classifyTransition(prev, next);
+      expect(shouldNotifyTransition(kind, policy)).toBe(expected);
+    });
+  }
+});

package/src/notification-policy.ts

@@ -0,0 +1,56 @@
+import type {
+  HealthCheckStatus,
+  NotificationPolicy,
+} from "@checkstack/healthcheck-common";
+
+/**
+ * The kind of transition a system health change represents. Used to
+ * decide whether a notification should fire and how its CTA should
+ * link back into the UI.
+ */
+export type TransitionKind =
+  /** No actual change (e.g. healthy → healthy). */
+  | "none"
+  /** Severity increased (healthy → degraded, degraded → unhealthy, ...). */
+  | "escalation"
+  /** Severity decreased but did not reach healthy (unhealthy → degraded). */
+  | "deescalation"
+  /** Returned to healthy from any non-healthy state. */
+  | "recovery";
+
+const SEVERITY: Record<HealthCheckStatus, number> = {
+  healthy: 0,
+  degraded: 1,
+  unhealthy: 2,
+};
+
+/**
+ * Classify a transition between two health states. Pure and total over
+ * the cartesian product of `HealthCheckStatus` values.
+ */
+export function classifyTransition(
+  previous: HealthCheckStatus,
+  next: HealthCheckStatus,
+): TransitionKind {
+  if (previous === next) return "none";
+  if (next === "healthy") return "recovery";
+  return SEVERITY[next] > SEVERITY[previous] ? "escalation" : "deescalation";
+}
+
+/**
+ * Decide whether a transition should produce a notification given the
+ * effective per-system policy. Escalations and recoveries always notify;
+ * de-escalations are suppressed when the policy opts in.
+ *
+ * Accepts the narrowed `Pick` because callers may only have the
+ * suppression flag — full policy resolution requires per-check lookups
+ * that aren't relevant to this decision.
+ */
+export function shouldNotifyTransition(
+  kind: TransitionKind,
+  policy: Pick<NotificationPolicy, "suppressDeEscalations">,
+): boolean {
+  if (kind === "none") return false;
+  if (kind === "deescalation" && policy.suppressDeEscalations) return false;
+  return true;
+}

package/src/queue-executor.test.ts

@@ -72,6 +72,7 @@ const createMockCatalogClient = () => ({
   // Other methods not used in queue-executor
   getEntities: mock(async () => ({ systems: [], groups: [] })),
   getSystems: mock(async () => ({ systems: [] })),
+  getSystem: mock(async () => null),
   getGroups: mock(async () => []),
   createSystem: mock(async () => ({})),
   updateSystem: mock(async () => ({})),
@@ -415,4 +416,140 @@ describe("Queue-Based Health Check Executor", () => {
       expect(mockSignalService.getRecordedSignals()).toHaveLength(0);
     });
   });
+
+  describe("executeHealthCheckJob - collector run-context", () => {
+    it("passes curated run-context to the collector (name falls back to id when configName is null)", async () => {
+      const mockDb = createMockDb();
+      const mockRegistry = createMockRegistry();
+      const mockLogger = createMockLogger();
+      const mockQueueManager = createMockQueueManager();
+      const mockCatalogClient = createMockCatalogClient();
+      const mockMaintenanceClient = createMockMaintenanceClient();
+      const mockIncidentClient = createMockIncidentClient();
+      const mockSignalService = createMockSignalService();
+
+      // Catalog resolves the system name.
+      (mockCatalogClient.getSystem as any) = mock(async () => ({
+        id: "system-1",
+        name: "web-01",
+      }));
+
+      // configName is null -> run-context check.name must fall back to id.
+      let selectCallCount = 0;
+      (mockDb.select as any) = mock(() => {
+        selectCallCount++;
+        if (selectCallCount === 2) {
+          return {
+            from: mock(() => ({
+              innerJoin: mock(() => ({
+                where: mock(() =>
+                  Promise.resolve([
+                    {
+                      configId: "config-1",
+                      configName: null,
+                      strategyId: "test-strategy",
+                      config: { timeout: 5000 },
+                      collectors: [
+                        { id: "col-1", collectorId: "test-collector", config: {} },
+                      ],
+                      interval: 45,
+                      enabled: true,
+                      paused: false,
+                      includeLocal: true,
+                      satelliteIds: [],
+                    },
+                  ]),
+                ),
+              })),
+            })),
+          };
+        }
+        return {
+          from: mock(() => ({
+            innerJoin: mock(() => ({
+              where: mock(() => Promise.resolve([])),
+            })),
+          })),
+        };
+      });
+
+      // Capture the run-context the collector receives.
+      let capturedRunContext: unknown;
+      const collectorExecute = mock(
+        async (params: { runContext?: unknown }) => {
+          capturedRunContext = params.runContext;
+          return { result: {} };
+        },
+      );
+      const mockCollectorRegistry = {
+        register: mock(() => {}),
+        getCollector: mock(() => ({
+          collector: {
+            id: "test-collector",
+            execute: collectorExecute,
+            mergeResult: mock(() => ({})),
+          },
+        })),
+        getCollectors: mock(() => []),
+      };
+
+      const queue =
+        mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
+      let capturedHandler:
+        | ((job: { data: HealthCheckJobPayload }) => Promise<void>)
+        | undefined;
+      (queue.consume as any) = mock(
+        async (
+          handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
+        ) => {
+          capturedHandler = handler;
+        },
+      );
+
+      await setupHealthCheckWorker({
+        db: mockDb as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["db"],
+        registry: mockRegistry,
+        collectorRegistry: mockCollectorRegistry as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["collectorRegistry"],
+        logger: mockLogger,
+        queueManager: mockQueueManager,
+        signalService: mockSignalService,
+        catalogClient: mockCatalogClient as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["catalogClient"],
+        notificationClient: {
+          notifyForSubscription: () => Promise.resolve({ notifiedCount: 0 }),
+        } as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["notificationClient"],
+        maintenanceClient: mockMaintenanceClient as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["maintenanceClient"],
+        incidentClient: mockIncidentClient as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["incidentClient"],
+        getEmitHook: () => undefined,
+        cache: passthroughCache,
+      });
+
+      if (capturedHandler) {
+        // The collector runs early in the execution sequence; downstream
+        // aggregation/persistence touches DB surfaces the lightweight mock
+        // doesn't model, so tolerate a later throw — the run-context we
+        // assert on is captured synchronously at collector-execute time.
+        await capturedHandler({
+          data: { configId: "config-1", systemId: "system-1" },
+        }).catch(() => {});
+      }
+
+      expect(collectorExecute).toHaveBeenCalled();
+      expect(capturedRunContext).toEqual({
+        check: { id: "config-1", name: "config-1", intervalSeconds: 45 },
+        system: { id: "system-1", name: "web-01" },
+      });
+    });
+  });
 });