@checkstack/healthcheck-backend 1.1.4 → 1.3.0

package/src/automations.ts

@@ -0,0 +1,340 @@
+/**
+ * Healthcheck triggers + actions registered with the Automation Platform.
+ *
+ * Triggers:
+ *   - `healthcheck.system.degraded` — existing directional hook
+ *   - `healthcheck.system.healthy` — existing directional hook
+ *   - `healthcheck.system.health_changed` — new umbrella hook,
+ *     fires on every aggregated-health transition. Carries both the
+ *     previous and new statuses so subscribers don't have to listen
+ *     to two hooks and coalesce themselves.
+ *
+ * Actions:
+ *   - `healthcheck.run_now`: enqueue a one-off run of a specific
+ *     `(systemId, configurationId)` assignment. The recurring
+ *     schedule keeps ticking; this just nudges the queue.
+ *   - `healthcheck.enable_assignment` /
+ *     `healthcheck.disable_assignment`: flip the `enabled` flag on an
+ *     existing assignment via `service.setAssignmentEnabled`. Emits
+ *     the existing `assignmentChanged` hook so the satellite-config
+ *     relay picks up the change.
+ *
+ * Mutation actions emit hooks themselves (via the `emitHook` factory
+ * dep) so downstream automations + caches react the same way as
+ * RPC-driven mutations.
+ */
+import { z } from "zod";
+import { Versioned, type Hook } from "@checkstack/backend-api";
+import type { QueueManager } from "@checkstack/queue-api";
+import type {
+  ActionDefinition,
+  TriggerDefinition,
+} from "@checkstack/automation-backend";
+import { HealthCheckStatusSchema } from "@checkstack/healthcheck-common";
+
+import { healthCheckHooks } from "./hooks";
+import {
+  HEALTH_CHECK_QUEUE,
+  type HealthCheckJobPayload,
+} from "./queue-executor";
+import type { HealthCheckService } from "./service";
+
+// ─── Payload schemas — match the hook payloads exactly ─────────────────
+
+const systemDegradedPayloadSchema = z.object({
+  systemId: z.string(),
+  systemName: z.string().optional(),
+  previousStatus: HealthCheckStatusSchema,
+  newStatus: HealthCheckStatusSchema,
+  healthyChecks: z.number(),
+  totalChecks: z.number(),
+  timestamp: z.string(),
+});
+
+const systemHealthyPayloadSchema = z.object({
+  systemId: z.string(),
+  systemName: z.string().optional(),
+  previousStatus: HealthCheckStatusSchema,
+  healthyChecks: z.number(),
+  totalChecks: z.number(),
+  timestamp: z.string(),
+});
+
+const systemHealthChangedPayloadSchema = z.object({
+  systemId: z.string(),
+  systemName: z.string().optional(),
+  previousStatus: HealthCheckStatusSchema,
+  newStatus: HealthCheckStatusSchema,
+  healthyChecks: z.number(),
+  totalChecks: z.number(),
+  timestamp: z.string(),
+});
+
+const checkFailedPayloadSchema = z.object({
+  systemId: z.string(),
+  configurationId: z.string(),
+  status: HealthCheckStatusSchema,
+  latencyMs: z.number().optional(),
+  result: z.record(z.string(), z.unknown()).optional(),
+  timestamp: z.string(),
+});
+
+const flappingDetectedPayloadSchema = z.object({
+  systemId: z.string(),
+  configurationId: z.string(),
+  transitionCount: z.number(),
+  windowMinutes: z.number(),
+  timestamp: z.string(),
+});
+
+// ─── Triggers ──────────────────────────────────────────────────────────
+
+export const systemDegradedTrigger: TriggerDefinition<
+  z.infer<typeof systemDegradedPayloadSchema>
+> = {
+  id: "system_degraded",
+  displayName: "System Health Degraded",
+  description:
+    "Fires when a system's health transitions from healthy to degraded/unhealthy",
+  category: "Health",
+  icon: "HeartPulse",
+  payloadSchema: systemDegradedPayloadSchema,
+  hook: healthCheckHooks.systemDegraded,
+  contextKey: (p) => p.systemId,
+};
+
+export const systemHealthyTrigger: TriggerDefinition<
+  z.infer<typeof systemHealthyPayloadSchema>
+> = {
+  id: "system_healthy",
+  displayName: "System Health Restored",
+  description: "Fires when a system's health recovers to healthy",
+  category: "Health",
+  icon: "HeartPulse",
+  payloadSchema: systemHealthyPayloadSchema,
+  hook: healthCheckHooks.systemHealthy,
+  contextKey: (p) => p.systemId,
+};
+
+export const systemHealthChangedTrigger: TriggerDefinition<
+  z.infer<typeof systemHealthChangedPayloadSchema>
+> = {
+  id: "system_health_changed",
+  displayName: "System Health Changed",
+  description:
+    "Fires on every aggregated-health transition — carries previous + new status",
+  category: "Health",
+  icon: "HeartPulse",
+  payloadSchema: systemHealthChangedPayloadSchema,
+  hook: healthCheckHooks.systemHealthChanged,
+  contextKey: (p) => p.systemId,
+};
+
+export const checkFailedTrigger: TriggerDefinition<
+  z.infer<typeof checkFailedPayloadSchema>
+> = {
+  id: "check_failed",
+  displayName: "Health Check Failed",
+  description:
+    "Fires when an individual check run completes with a non-`healthy` status",
+  category: "Health",
+  icon: "TriangleAlert",
+  payloadSchema: checkFailedPayloadSchema,
+  hook: healthCheckHooks.checkFailed,
+  contextKey: (p) => p.systemId,
+};
+
+export const flappingDetectedTrigger: TriggerDefinition<
+  z.infer<typeof flappingDetectedPayloadSchema>
+> = {
+  id: "flapping_detected",
+  displayName: "Health Check Flapping",
+  description:
+    "Fires when N unhealthy transitions are observed within the policy window. Re-fires on every additional transition while flapping; debounce in the automation if needed.",
+  category: "Health",
+  icon: "Repeat",
+  payloadSchema: flappingDetectedPayloadSchema,
+  hook: healthCheckHooks.flappingDetected,
+  contextKey: (p) => p.systemId,
+};
+
+export const healthCheckTriggers: TriggerDefinition<unknown>[] = [
+  systemDegradedTrigger as TriggerDefinition<unknown>,
+  systemHealthyTrigger as TriggerDefinition<unknown>,
+  systemHealthChangedTrigger as TriggerDefinition<unknown>,
+  checkFailedTrigger as TriggerDefinition<unknown>,
+  flappingDetectedTrigger as TriggerDefinition<unknown>,
+];
+
+// ─── Action configs ────────────────────────────────────────────────────
+
+const runNowConfigSchema = z.object({
+  systemId: z.string().min(1).describe("Target system id"),
+  configurationId: z
+    .string()
+    .min(1)
+    .describe("Target health-check configuration id"),
+});
+
+const assignmentToggleConfigSchema = z.object({
+  systemId: z.string().min(1),
+  configurationId: z.string().min(1),
+});
+
+// ─── Artifact ──────────────────────────────────────────────────────────
+
+const assignmentArtifactSchema = z.object({
+  systemId: z.string(),
+  configurationId: z.string(),
+  enabled: z.boolean().optional(),
+  enqueued: z.boolean().optional(),
+});
+
+export type AssignmentArtifact = z.infer<typeof assignmentArtifactSchema>;
+
+export const assignmentArtifactType = {
+  id: "assignment",
+  displayName: "Healthcheck Assignment",
+  description:
+    "Identifies the system↔configuration assignment touched by an automation action",
+  schema: assignmentArtifactSchema,
+} as const;
+
+// ─── Action factory ────────────────────────────────────────────────────
+
+export interface HealthCheckActionDeps {
+  service: HealthCheckService;
+  queueManager: QueueManager;
+  emitHook: <T>(hook: Hook<T>, payload: T) => Promise<void>;
+}
+
+export function createHealthCheckActions(
+  deps: HealthCheckActionDeps,
+): ActionDefinition<unknown, unknown>[] {
+  const runNow: ActionDefinition<
+    z.infer<typeof runNowConfigSchema>,
+    AssignmentArtifact
+  > = {
+    id: "run_now",
+    displayName: "Run Health Check Now",
+    description:
+      "Enqueue a one-off run of the given assignment. Doesn't disturb the recurring schedule.",
+    category: "Health",
+    icon: "Play",
+    config: new Versioned({ version: 1, schema: runNowConfigSchema }),
+    produces: "healthcheck.assignment",
+    execute: async ({ config, logger }) => {
+      const queue = deps.queueManager.getQueue<HealthCheckJobPayload>(
+        HEALTH_CHECK_QUEUE,
+      );
+      await queue.enqueue({
+        configId: config.configurationId,
+        systemId: config.systemId,
+      });
+      logger.info(
+        `Automation enqueued run for ${config.systemId}:${config.configurationId}`,
+      );
+      return {
+        success: true,
+        externalId: `${config.systemId}:${config.configurationId}`,
+        artifact: {
+          systemId: config.systemId,
+          configurationId: config.configurationId,
+          enqueued: true,
+        },
+      };
+    },
+  };
+
+  const enableAssignment: ActionDefinition<
+    z.infer<typeof assignmentToggleConfigSchema>,
+    AssignmentArtifact
+  > = {
+    id: "enable_assignment",
+    displayName: "Enable Health Check Assignment",
+    description:
+      "Flip the `enabled` flag on an existing system↔configuration assignment to true.",
+    category: "Health",
+    icon: "Power",
+    config: new Versioned({ version: 1, schema: assignmentToggleConfigSchema }),
+    produces: "healthcheck.assignment",
+    execute: async ({ config, logger }) => {
+      const updated = await deps.service.setAssignmentEnabled(
+        config.systemId,
+        config.configurationId,
+        true,
+      );
+      if (!updated) {
+        return {
+          success: false,
+          error: `Assignment not found: ${config.systemId} ↔ ${config.configurationId}`,
+        };
+      }
+      await deps.emitHook(healthCheckHooks.assignmentChanged, {
+        systemId: config.systemId,
+        configurationId: config.configurationId,
+      });
+      logger.info(
+        `Automation enabled assignment ${config.systemId}:${config.configurationId}`,
+      );
+      return {
+        success: true,
+        externalId: `${config.systemId}:${config.configurationId}`,
+        artifact: {
+          systemId: config.systemId,
+          configurationId: config.configurationId,
+          enabled: true,
+        },
+      };
+    },
+  };
+
+  const disableAssignment: ActionDefinition<
+    z.infer<typeof assignmentToggleConfigSchema>,
+    AssignmentArtifact
+  > = {
+    id: "disable_assignment",
+    displayName: "Disable Health Check Assignment",
+    description:
+      "Flip the `enabled` flag on an existing system↔configuration assignment to false.",
+    category: "Health",
+    icon: "PowerOff",
+    config: new Versioned({ version: 1, schema: assignmentToggleConfigSchema }),
+    produces: "healthcheck.assignment",
+    execute: async ({ config, logger }) => {
+      const updated = await deps.service.setAssignmentEnabled(
+        config.systemId,
+        config.configurationId,
+        false,
+      );
+      if (!updated) {
+        return {
+          success: false,
+          error: `Assignment not found: ${config.systemId} ↔ ${config.configurationId}`,
+        };
+      }
+      await deps.emitHook(healthCheckHooks.assignmentChanged, {
+        systemId: config.systemId,
+        configurationId: config.configurationId,
+      });
+      logger.info(
+        `Automation disabled assignment ${config.systemId}:${config.configurationId}`,
+      );
+      return {
+        success: true,
+        externalId: `${config.systemId}:${config.configurationId}`,
+        artifact: {
+          systemId: config.systemId,
+          configurationId: config.configurationId,
+          enabled: false,
+        },
+      };
+    },
+  };
+
+  return [
+    runNow as ActionDefinition<unknown, unknown>,
+    enableAssignment as ActionDefinition<unknown, unknown>,
+    disableAssignment as ActionDefinition<unknown, unknown>,
+  ];
+}

package/src/healthcheck-gitops-kinds.test.ts

@@ -38,6 +38,22 @@ interface MockAssociation {
   systemId: string;
   configurationId: string;
   enabled: boolean;
+  notificationPolicy?: {
+    suppressDeEscalations: boolean;
+    autoOpenIncidentOnUnhealthy: boolean;
+    useNotificationSuppression: boolean;
+    skipDuringMaintenance: boolean;
+    sustainedUnhealthyTrigger: {
+      enabled: boolean;
+      durationMinutes: number;
+    };
+    flappingTrigger: {
+      enabled: boolean;
+      transitions: number;
+      windowMinutes: number;
+    };
+    autoCloseAfterMinutes: number | null;
+  };
 }
 
 function createMockService() {
@@ -80,6 +96,7 @@ function createMockService() {
         systemId: string;
         configurationId: string;
         enabled?: boolean;
+        notificationPolicy?: MockAssociation["notificationPolicy"];
       }) => {
         const existing = associations.find(
           (a) =>
@@ -88,11 +105,13 @@ function createMockService() {
         );
         if (existing) {
           existing.enabled = props.enabled ?? true;
+          existing.notificationPolicy = props.notificationPolicy;
         } else {
           associations.push({
             systemId: props.systemId,
             configurationId: props.configurationId,
             enabled: props.enabled ?? true,
+            notificationPolicy: props.notificationPolicy,
           });
         }
       },
@@ -619,6 +638,80 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
     ).rejects.toThrow(/Cannot resolve Healthcheck ref "nonexistent-check"/);
   });
 
+  it("passes a fully-defaulted notificationPolicy through when partial fields are supplied", async () => {
+    const ext = buildExtension();
+
+    const contextWithRefs: ReconcileContext = {
+      ...mockContext,
+      resolveEntityRef: async ({ kind, entityName }) =>
+        kind === "Healthcheck" && entityName === "db-check" ? "hc-1" : undefined,
+    };
+
+    await ext.reconcile({
+      entity: {
+        apiVersion: CHECKSTACK_API_VERSION,
+        kind: "System",
+        metadata: { name: "payment-service" },
+        spec: {},
+      },
+      extensionSpec: [
+        {
+          ref: { kind: "Healthcheck", name: "db-check" },
+          // Operator only sets the flap threshold and disables
+          // auto-close; everything else should default in via the
+          // schema parse.
+          notificationPolicy: {
+            flappingTrigger: { transitions: 5 },
+            autoCloseAfterMinutes: null,
+          },
+        },
+      ],
+      entityId: "sys-123",
+      context: contextWithRefs,
+    });
+
+    const policy = mockService.associations[0]?.notificationPolicy;
+    expect(policy).toBeDefined();
+    expect(policy?.suppressDeEscalations).toBe(false);
+    expect(policy?.autoOpenIncidentOnUnhealthy).toBe(true);
+    expect(policy?.useNotificationSuppression).toBe(true);
+    expect(policy?.skipDuringMaintenance).toBe(true);
+    expect(policy?.sustainedUnhealthyTrigger).toEqual({
+      enabled: true,
+      durationMinutes: 30,
+    });
+    expect(policy?.flappingTrigger).toEqual({
+      enabled: true,
+      transitions: 5,
+      windowMinutes: 60,
+    });
+    expect(policy?.autoCloseAfterMinutes).toBeNull();
+  });
+
+  it("omits notificationPolicy entirely when the spec doesn't set it", async () => {
+    const ext = buildExtension();
+
+    const contextWithRefs: ReconcileContext = {
+      ...mockContext,
+      resolveEntityRef: async ({ kind, entityName }) =>
+        kind === "Healthcheck" && entityName === "db-check" ? "hc-1" : undefined,
+    };
+
+    await ext.reconcile({
+      entity: {
+        apiVersion: CHECKSTACK_API_VERSION,
+        kind: "System",
+        metadata: { name: "payment-service" },
+        spec: {},
+      },
+      extensionSpec: [{ ref: { kind: "Healthcheck", name: "db-check" } }],
+      entityId: "sys-123",
+      context: contextWithRefs,
+    });
+
+    expect(mockService.associations[0]?.notificationPolicy).toBeUndefined();
+  });
+
   it("skips when extensionSpec is empty", async () => {
     const ext = buildExtension();
 

package/src/healthcheck-gitops-kinds.ts

@@ -13,6 +13,7 @@ import type {
   HealthCheckRegistry,
   CollectorRegistry,
 } from "@checkstack/backend-api";
+import { NotificationPolicySchema } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
 import {
   DynamicOperators,
@@ -81,6 +82,29 @@ const systemHealthcheckExtensionSchema = z
       unhealthyThreshold: z.number().int().min(1).optional(),
       satelliteIds: z.array(z.string()).optional(),
       includeLocal: z.boolean().optional(),
+      /**
+       * Per-assignment notification policy. Any field omitted falls
+       * back to the platform default (see `DEFAULT_NOTIFICATION_POLICY`).
+       * Inner objects (`sustainedUnhealthyTrigger`, `flappingTrigger`)
+       * are also accepted partially.
+       */
+      notificationPolicy: NotificationPolicySchema.partial()
+        .extend({
+          sustainedUnhealthyTrigger: z
+            .object({
+              enabled: z.boolean().optional(),
+              durationMinutes: z.number().int().min(1).optional(),
+            })
+            .optional(),
+          flappingTrigger: z
+            .object({
+              enabled: z.boolean().optional(),
+              transitions: z.number().int().min(1).optional(),
+              windowMinutes: z.number().int().min(1).optional(),
+            })
+            .optional(),
+        })
+        .optional(),
     }),
   )
   .optional();
@@ -317,6 +341,15 @@ export function buildSystemHealthcheckExtension(
               }
             : undefined;
 
+        // Materialise the (possibly partial) policy through the full
+        // schema so DEFAULT_NOTIFICATION_POLICY fields fill in any
+        // keys the operator omitted. Omitting `notificationPolicy`
+        // entirely leaves the stored value as null (defaults applied
+        // at read time).
+        const notificationPolicy = entry.notificationPolicy
+          ? NotificationPolicySchema.parse(entry.notificationPolicy)
+          : undefined;
+
         await service.associateSystem({
           systemId: systemEntityId,
           configurationId: configId,
@@ -324,6 +357,7 @@ export function buildSystemHealthcheckExtension(
           stateThresholds,
           satelliteIds: entry.satelliteIds,
           includeLocal: entry.includeLocal,
+          notificationPolicy,
         });
 
         // Retrieve config to get the interval for scheduling

package/src/hooks.ts

@@ -1,8 +1,14 @@
 import { createHook } from "@checkstack/backend-api";
+import type { HealthCheckStatus } from "@checkstack/healthcheck-common";
 
 /**
  * Health check hooks for cross-plugin communication and external integrations.
  * These hooks are registered as integration events for webhook subscriptions.
+ *
+ * `status` / `previousStatus` / `newStatus` carry the canonical
+ * `HealthCheckStatus` enum values, so automation triggers built on
+ * these hooks can offer the known values for `==` comparisons in the
+ * editor.
  */
 export const healthCheckHooks = {
   /**
@@ -13,8 +19,8 @@ export const healthCheckHooks = {
   systemDegraded: createHook<{
     systemId: string;
     systemName?: string;
-    previousStatus: string;
-    newStatus: string;
+    previousStatus: HealthCheckStatus;
+    newStatus: HealthCheckStatus;
     healthyChecks: number;
     totalChecks: number;
     timestamp: string;
@@ -27,7 +33,7 @@ export const healthCheckHooks = {
   systemHealthy: createHook<{
     systemId: string;
     systemName?: string;
-    previousStatus: string;
+    previousStatus: HealthCheckStatus;
     healthyChecks: number;
     totalChecks: number;
     timestamp: string;
@@ -50,9 +56,68 @@ export const healthCheckHooks = {
   checkCompleted: createHook<{
     systemId: string;
     configurationId: string;
-    status: string;
+    status: HealthCheckStatus;
     latencyMs: number | undefined;
     result: Record<string, unknown> | undefined;
     timestamp: string;
   }>("healthcheck.check.completed"),
+
+  /**
+   * Umbrella variant of `systemDegraded` + `systemHealthy` — fires on
+   * **any** aggregated-health transition, carrying both the previous
+   * and new statuses. Subscribers (e.g. an automation that wants to
+   * react to every state change without subscribing to two hooks
+   * and coalescing themselves) prefer this one.
+   *
+   * Emitted alongside the directional hooks, never instead of them,
+   * so existing subscribers keep working unchanged.
+   */
+  systemHealthChanged: createHook<{
+    systemId: string;
+    systemName?: string;
+    previousStatus: HealthCheckStatus;
+    newStatus: HealthCheckStatus;
+    healthyChecks: number;
+    totalChecks: number;
+    timestamp: string;
+  }>("healthcheck.system.health_changed"),
+
+  /**
+   * Narrow variant of `checkCompleted` — fires only when an individual
+   * check run completed with a non-`healthy` status. Carries the
+   * latency + raw result so subscribers can branch on collector-
+   * specific fields without re-querying. Operators usually prefer
+   * this over `checkCompleted` for incident-style automation because
+   * a "trigger on any completion, then filter" automation is harder
+   * to read at a glance than a typed `check_failed` entry point.
+   */
+  checkFailed: createHook<{
+    systemId: string;
+    configurationId: string;
+    status: HealthCheckStatus;
+    latencyMs: number | undefined;
+    result: Record<string, unknown> | undefined;
+    timestamp: string;
+  }>("healthcheck.check.failed"),
+
+  /**
+   * Emitted when the flapping-detector observes ≥ N unhealthy
+   * transitions in the policy's configured window. Fires regardless
+   * of whether `autoOpenIncidentOnUnhealthy` is enabled — the hook is
+   * informational; the auto-incident pipeline still gates on the
+   * policy.
+   *
+   * Re-fires on every additional transition past the threshold while
+   * the check stays in a flapping pattern, so automations that want
+   * "page once and only once" should debounce on `(systemId,
+   * configurationId)`. Carrying the observed transition count + the
+   * window length lets subscribers reason about both.
+   */
+  flappingDetected: createHook<{
+    systemId: string;
+    configurationId: string;
+    transitionCount: number;
+    windowMinutes: number;
+    timestamp: string;
+  }>("healthcheck.flapping_detected"),
 } as const;