@checkstack/healthcheck-backend 1.1.4 → 1.2.0

package/src/auto-incident.ts

@@ -0,0 +1,332 @@
+import { and, desc, eq, gte, isNotNull, isNull, sql } from "drizzle-orm";
+import type {
+  HealthCheckStatus,
+  NotificationPolicy,
+} from "@checkstack/healthcheck-common";
+import type { Logger, SafeDatabase } from "@checkstack/backend-api";
+import type { InferClient } from "@checkstack/common";
+import { IncidentApi } from "@checkstack/incident-common";
+import { MaintenanceApi } from "@checkstack/maintenance-common";
+import {
+  healthCheckAutoIncidents,
+  healthCheckRuns,
+  healthCheckUnhealthyTransitions,
+} from "./schema";
+import * as schema from "./schema";
+
+type Db = SafeDatabase<typeof schema>;
+type IncidentClient = InferClient<typeof IncidentApi>;
+type MaintenanceClient = InferClient<typeof MaintenanceApi>;
+
+/**
+ * Returns true when the per-check evaluated state went from anything
+ * other than `unhealthy` to `unhealthy` between two evaluations.
+ */
+export function isTransitionToUnhealthy(
+  previous: HealthCheckStatus | undefined,
+  next: HealthCheckStatus,
+): boolean {
+  return next === "unhealthy" && previous !== "unhealthy";
+}
+
+/**
+ * Record a transition-to-unhealthy in the audit table and return the
+ * total transition count for this check inside the configured window
+ * (the new row is included in the count). When `since` is provided,
+ * only transitions strictly after that timestamp are counted — used
+ * to ensure a freshly-opened auto-incident isn't re-triggered by
+ * pre-close transitions after the prior incident was resolved.
+ */
+export async function recordUnhealthyTransition({
+  db,
+  configurationId,
+  systemId,
+  windowMinutes,
+  since,
+  now = new Date(),
+}: {
+  db: Db;
+  configurationId: string;
+  systemId: string;
+  windowMinutes: number;
+  since?: Date;
+  now?: Date;
+}): Promise<number> {
+  await db.insert(healthCheckUnhealthyTransitions).values({
+    configurationId,
+    systemId,
+    transitionedAt: now,
+  });
+
+  const windowStart = new Date(now.getTime() - windowMinutes * 60_000);
+  const lowerBound =
+    since && since > windowStart ? since : windowStart;
+
+  const result = await db
+    .select({ count: sql<number>`COUNT(*)::int` })
+    .from(healthCheckUnhealthyTransitions)
+    .where(
+      and(
+        eq(healthCheckUnhealthyTransitions.configurationId, configurationId),
+        eq(healthCheckUnhealthyTransitions.systemId, systemId),
+        gte(healthCheckUnhealthyTransitions.transitionedAt, lowerBound),
+      ),
+    );
+
+  return result[0]?.count ?? 0;
+}
+
+/**
+ * Decide whether the flapping trigger should open an auto-incident.
+ * Returns false when the trigger is disabled or the count is below
+ * the configured threshold.
+ */
+export function shouldOpenForFlapping({
+  policy,
+  recentTransitionCount,
+}: {
+  policy: NotificationPolicy;
+  recentTransitionCount: number;
+}): boolean {
+  if (!policy.autoOpenIncidentOnUnhealthy) return false;
+  if (!policy.flappingTrigger.enabled) return false;
+  return recentTransitionCount >= policy.flappingTrigger.transitions;
+}
+
+/**
+ * Decide whether the sustained-duration trigger should open an
+ * auto-incident given the elapsed-unhealthy time for this check.
+ */
+export function shouldOpenForSustainedUnhealthy({
+  policy,
+  unhealthyForMs,
+}: {
+  policy: NotificationPolicy;
+  /** How long the check has been continuously unhealthy. */
+  unhealthyForMs: number;
+}): boolean {
+  if (!policy.autoOpenIncidentOnUnhealthy) return false;
+  if (!policy.sustainedUnhealthyTrigger.enabled) return false;
+  const thresholdMs =
+    policy.sustainedUnhealthyTrigger.durationMinutes * 60_000;
+  return unhealthyForMs >= thresholdMs;
+}
+
+/**
+ * Find the most recent transition to `unhealthy` for this check that
+ * happened after `since` (if provided). Used by the sustained-trigger
+ * evaluator to compute "how long has the check been unhealthy?"
+ */
+export async function findUnhealthySince({
+  db,
+  configurationId,
+  systemId,
+  since,
+}: {
+  db: Db;
+  configurationId: string;
+  systemId: string;
+  since?: Date;
+}): Promise<Date | undefined> {
+  const conditions = [
+    eq(healthCheckUnhealthyTransitions.configurationId, configurationId),
+    eq(healthCheckUnhealthyTransitions.systemId, systemId),
+  ];
+  if (since) {
+    conditions.push(gte(healthCheckUnhealthyTransitions.transitionedAt, since));
+  }
+
+  const [row] = await db
+    .select({
+      transitionedAt: healthCheckUnhealthyTransitions.transitionedAt,
+    })
+    .from(healthCheckUnhealthyTransitions)
+    .where(and(...conditions))
+    .orderBy(desc(healthCheckUnhealthyTransitions.transitionedAt))
+    .limit(1);
+
+  return row?.transitionedAt;
+}
+
+/**
+ * Find any currently-active (closedAt IS NULL) auto-incident for the
+ * system. Used to avoid opening a duplicate when one is already open.
+ */
+export async function findActiveAutoIncident({
+  db,
+  systemId,
+}: {
+  db: Db;
+  systemId: string;
+}): Promise<{ id: string; incidentId: string } | undefined> {
+  const rows = await db
+    .select({
+      id: healthCheckAutoIncidents.id,
+      incidentId: healthCheckAutoIncidents.incidentId,
+    })
+    .from(healthCheckAutoIncidents)
+    .where(
+      and(
+        eq(healthCheckAutoIncidents.systemId, systemId),
+        isNull(healthCheckAutoIncidents.closedAt),
+      ),
+    )
+    .limit(1);
+
+  return rows[0];
+}
+
+/**
+ * Most recent close time for an auto-incident on this assignment, or
+ * undefined if none has ever closed. Used to gate re-opens behind a
+ * "must recover first" rule.
+ */
+export async function findLastAutoIncidentClose({
+  db,
+  systemId,
+  configurationId,
+}: {
+  db: Db;
+  systemId: string;
+  configurationId: string;
+}): Promise<Date | undefined> {
+  const [row] = await db
+    .select({ closedAt: healthCheckAutoIncidents.closedAt })
+    .from(healthCheckAutoIncidents)
+    .where(
+      and(
+        eq(healthCheckAutoIncidents.systemId, systemId),
+        eq(healthCheckAutoIncidents.configurationId, configurationId),
+        isNotNull(healthCheckAutoIncidents.closedAt),
+      ),
+    )
+    .orderBy(desc(healthCheckAutoIncidents.closedAt))
+    .limit(1);
+
+  return row?.closedAt ?? undefined;
+}
+
+/**
+ * Has this check produced at least one healthy run since the given
+ * timestamp? Used to confirm the system has actually recovered between
+ * the last auto-incident close and now before a new auto-incident is
+ * allowed to open.
+ */
+export async function hasHealthyRunSince({
+  db,
+  systemId,
+  configurationId,
+  since,
+}: {
+  db: Db;
+  systemId: string;
+  configurationId: string;
+  since: Date;
+}): Promise<boolean> {
+  const [row] = await db
+    .select({ id: healthCheckRuns.id })
+    .from(healthCheckRuns)
+    .where(
+      and(
+        eq(healthCheckRuns.systemId, systemId),
+        eq(healthCheckRuns.configurationId, configurationId),
+        eq(healthCheckRuns.status, "healthy"),
+        gte(healthCheckRuns.timestamp, since),
+      ),
+    )
+    .limit(1);
+
+  return !!row;
+}
+
+/**
+ * Check whether the system currently has an active maintenance window
+ * with suppression. Falls back to "not suppressed" on errors so a
+ * downstream outage doesn't accidentally block legitimate incidents.
+ */
+export async function isMaintenanceSuppressed({
+  maintenanceClient,
+  systemId,
+  logger,
+}: {
+  maintenanceClient: MaintenanceClient;
+  systemId: string;
+  logger: Logger;
+}): Promise<boolean> {
+  try {
+    const { suppressed } =
+      await maintenanceClient.hasActiveMaintenanceWithSuppression({ systemId });
+    return suppressed;
+  } catch (error) {
+    logger.warn(
+      `Failed to check maintenance for ${systemId} during auto-incident decision; assuming not suppressed:`,
+      error,
+    );
+    return false;
+  }
+}
+
+/**
+ * Open an auto-incident through the incident plugin's service-level
+ * RPC and persist the mapping so the auto-close worker can find and
+ * resolve it later. No-op (returns existing mapping) when an active
+ * auto-incident already exists for the system.
+ */
+export async function openAutoIncident({
+  db,
+  incidentClient,
+  logger,
+  systemId,
+  systemName,
+  configurationId,
+  configurationName,
+  policy,
+  reason,
+}: {
+  db: Db;
+  incidentClient: IncidentClient;
+  logger: Logger;
+  systemId: string;
+  systemName: string;
+  configurationId: string;
+  configurationName: string;
+  policy: NotificationPolicy;
+  /** Short human-readable phrase for the incident description. */
+  reason: string;
+}): Promise<{ incidentId: string } | undefined> {
+  const existing = await findActiveAutoIncident({ db, systemId });
+  if (existing) {
+    return { incidentId: existing.incidentId };
+  }
+
+  try {
+    const { id: incidentId } = await incidentClient.createAutoIncident({
+      title: `${systemName} is critical`,
+      description: `Auto-opened by health check **${configurationName}** (${reason}).`,
+      severity: "critical",
+      suppressNotifications: policy.useNotificationSuppression,
+      systemIds: [systemId],
+      initialMessage: `Health check \`${configurationName}\` triggered the auto-incident: ${reason}.`,
+    });
+
+    await db.insert(healthCheckAutoIncidents).values({
+      incidentId,
+      systemId,
+      configurationId,
+      cooldownMinutes: policy.autoCloseAfterMinutes,
+    });
+
+    logger.info(
+      `Auto-opened incident ${incidentId} for system ${systemId} (check ${configurationId}; ${reason})`,
+    );
+    return { incidentId };
+  } catch (error) {
+    // Auto-incident creation is best-effort — failure here shouldn't
+    // block the rest of the health-check flow.
+    logger.warn(
+      `Failed to open auto-incident for system ${systemId} (check ${configurationId}):`,
+      error,
+    );
+    return undefined;
+  }
+}

package/src/healthcheck-gitops-kinds.test.ts

@@ -38,6 +38,22 @@ interface MockAssociation {
   systemId: string;
   configurationId: string;
   enabled: boolean;
+  notificationPolicy?: {
+    suppressDeEscalations: boolean;
+    autoOpenIncidentOnUnhealthy: boolean;
+    useNotificationSuppression: boolean;
+    skipDuringMaintenance: boolean;
+    sustainedUnhealthyTrigger: {
+      enabled: boolean;
+      durationMinutes: number;
+    };
+    flappingTrigger: {
+      enabled: boolean;
+      transitions: number;
+      windowMinutes: number;
+    };
+    autoCloseAfterMinutes: number | null;
+  };
 }
 
 function createMockService() {
@@ -80,6 +96,7 @@ function createMockService() {
         systemId: string;
         configurationId: string;
         enabled?: boolean;
+        notificationPolicy?: MockAssociation["notificationPolicy"];
       }) => {
         const existing = associations.find(
           (a) =>
@@ -88,11 +105,13 @@ function createMockService() {
         );
         if (existing) {
           existing.enabled = props.enabled ?? true;
+          existing.notificationPolicy = props.notificationPolicy;
         } else {
           associations.push({
             systemId: props.systemId,
             configurationId: props.configurationId,
             enabled: props.enabled ?? true,
+            notificationPolicy: props.notificationPolicy,
           });
         }
       },
@@ -619,6 +638,80 @@ describe("Healthcheck GitOps Kind: System Extension", () => {
     ).rejects.toThrow(/Cannot resolve Healthcheck ref "nonexistent-check"/);
   });
 
+  it("passes a fully-defaulted notificationPolicy through when partial fields are supplied", async () => {
+    const ext = buildExtension();
+
+    const contextWithRefs: ReconcileContext = {
+      ...mockContext,
+      resolveEntityRef: async ({ kind, entityName }) =>
+        kind === "Healthcheck" && entityName === "db-check" ? "hc-1" : undefined,
+    };
+
+    await ext.reconcile({
+      entity: {
+        apiVersion: CHECKSTACK_API_VERSION,
+        kind: "System",
+        metadata: { name: "payment-service" },
+        spec: {},
+      },
+      extensionSpec: [
+        {
+          ref: { kind: "Healthcheck", name: "db-check" },
+          // Operator only sets the flap threshold and disables
+          // auto-close; everything else should default in via the
+          // schema parse.
+          notificationPolicy: {
+            flappingTrigger: { transitions: 5 },
+            autoCloseAfterMinutes: null,
+          },
+        },
+      ],
+      entityId: "sys-123",
+      context: contextWithRefs,
+    });
+
+    const policy = mockService.associations[0]?.notificationPolicy;
+    expect(policy).toBeDefined();
+    expect(policy?.suppressDeEscalations).toBe(false);
+    expect(policy?.autoOpenIncidentOnUnhealthy).toBe(true);
+    expect(policy?.useNotificationSuppression).toBe(true);
+    expect(policy?.skipDuringMaintenance).toBe(true);
+    expect(policy?.sustainedUnhealthyTrigger).toEqual({
+      enabled: true,
+      durationMinutes: 30,
+    });
+    expect(policy?.flappingTrigger).toEqual({
+      enabled: true,
+      transitions: 5,
+      windowMinutes: 60,
+    });
+    expect(policy?.autoCloseAfterMinutes).toBeNull();
+  });
+
+  it("omits notificationPolicy entirely when the spec doesn't set it", async () => {
+    const ext = buildExtension();
+
+    const contextWithRefs: ReconcileContext = {
+      ...mockContext,
+      resolveEntityRef: async ({ kind, entityName }) =>
+        kind === "Healthcheck" && entityName === "db-check" ? "hc-1" : undefined,
+    };
+
+    await ext.reconcile({
+      entity: {
+        apiVersion: CHECKSTACK_API_VERSION,
+        kind: "System",
+        metadata: { name: "payment-service" },
+        spec: {},
+      },
+      extensionSpec: [{ ref: { kind: "Healthcheck", name: "db-check" } }],
+      entityId: "sys-123",
+      context: contextWithRefs,
+    });
+
+    expect(mockService.associations[0]?.notificationPolicy).toBeUndefined();
+  });
+
   it("skips when extensionSpec is empty", async () => {
     const ext = buildExtension();
 

package/src/healthcheck-gitops-kinds.ts

@@ -13,6 +13,7 @@ import type {
   HealthCheckRegistry,
   CollectorRegistry,
 } from "@checkstack/backend-api";
+import { NotificationPolicySchema } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
 import {
   DynamicOperators,
@@ -81,6 +82,29 @@ const systemHealthcheckExtensionSchema = z
       unhealthyThreshold: z.number().int().min(1).optional(),
       satelliteIds: z.array(z.string()).optional(),
       includeLocal: z.boolean().optional(),
+      /**
+       * Per-assignment notification policy. Any field omitted falls
+       * back to the platform default (see `DEFAULT_NOTIFICATION_POLICY`).
+       * Inner objects (`sustainedUnhealthyTrigger`, `flappingTrigger`)
+       * are also accepted partially.
+       */
+      notificationPolicy: NotificationPolicySchema.partial()
+        .extend({
+          sustainedUnhealthyTrigger: z
+            .object({
+              enabled: z.boolean().optional(),
+              durationMinutes: z.number().int().min(1).optional(),
+            })
+            .optional(),
+          flappingTrigger: z
+            .object({
+              enabled: z.boolean().optional(),
+              transitions: z.number().int().min(1).optional(),
+              windowMinutes: z.number().int().min(1).optional(),
+            })
+            .optional(),
+        })
+        .optional(),
     }),
   )
   .optional();
@@ -317,6 +341,15 @@ export function buildSystemHealthcheckExtension(
               }
             : undefined;
 
+        // Materialise the (possibly partial) policy through the full
+        // schema so DEFAULT_NOTIFICATION_POLICY fields fill in any
+        // keys the operator omitted. Omitting `notificationPolicy`
+        // entirely leaves the stored value as null (defaults applied
+        // at read time).
+        const notificationPolicy = entry.notificationPolicy
+          ? NotificationPolicySchema.parse(entry.notificationPolicy)
+          : undefined;
+
         await service.associateSystem({
           systemId: systemEntityId,
           configurationId: configId,
@@ -324,6 +357,7 @@ export function buildSystemHealthcheckExtension(
           stateThresholds,
           satelliteIds: entry.satelliteIds,
           includeLocal: entry.includeLocal,
+          notificationPolicy,
         });
 
         // Retrieve config to get the interval for scheduling

package/src/index.ts

@@ -3,6 +3,7 @@ import {
   bootstrapHealthChecks,
 } from "./queue-executor";
 import { setupRetentionJob } from "./retention-job";
+import { setupAutoIncidentCloseJob } from "./auto-incident-close-job";
 import * as schema from "./schema";
 import {
   healthCheckAccessRules,
@@ -34,6 +35,9 @@ import { HealthCheckService } from "./service";
 import { registerHealthcheckGitOpsKinds, registerHealthcheckGitOpsDocumentation } from "./healthcheck-gitops-kinds";
 import { catalogHooks } from "@checkstack/catalog-backend";
 import { satelliteHooks } from "@checkstack/satellite-backend";
+import { incidentHooks } from "@checkstack/incident-backend";
+import { eq, and, isNull } from "drizzle-orm";
+import { healthCheckAutoIncidents } from "./schema";
 import { CatalogApi } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import { IncidentApi } from "@checkstack/incident-common";
@@ -159,6 +163,7 @@ export default createBackendPlugin({
         queueManager: coreServices.queueManager,
         signalService: coreServices.signalService,
         cacheManager: coreServices.cacheManager,
+        config: coreServices.config,
       },
       // Phase 2: Register router and setup worker
       init: async ({
@@ -171,6 +176,7 @@ export default createBackendPlugin({
         queueManager,
         signalService,
         cacheManager,
+        config,
       }) => {
         logger.debug("🏥 Initializing Health Check Backend...");
 
@@ -225,6 +231,16 @@ export default createBackendPlugin({
           queueManager,
         });
 
+        // Setup auto-incident close worker (ticks every 60s, closes
+        // auto-opened incidents whose systems have been steady-healthy
+        // for the cooldown).
+        await setupAutoIncidentCloseJob({
+          db: database,
+          logger,
+          queueManager,
+          incidentClient,
+        });
+
         const healthCheckRouter = createHealthCheckRouter({
           database: database as SafeDatabase<typeof schema>,
           registry: healthCheckRegistry,
@@ -232,6 +248,7 @@ export default createBackendPlugin({
           gitOpsClient,
           getEmitHook: () => storedEmitHook,
           cache,
+          configService: config,
         });
         rpc.registerRouter(healthCheckRouter, healthCheckContract);
 
@@ -335,6 +352,32 @@ export default createBackendPlugin({
           { mode: "work-queue", workerGroup: "satellite-cleanup" },
         );
 
+        // Sync our auto-incident mapping when an incident is resolved.
+        // Without this, a manually-closed incident would still appear
+        // "active" in our mapping, blocking the require-recovery rule
+        // from re-evaluating fresh transitions.
+        onHook(
+          incidentHooks.incidentResolved,
+          async ({ incidentId }) => {
+            const updated = await database
+              .update(healthCheckAutoIncidents)
+              .set({ closedAt: new Date() })
+              .where(
+                and(
+                  eq(healthCheckAutoIncidents.incidentId, incidentId),
+                  isNull(healthCheckAutoIncidents.closedAt),
+                ),
+              )
+              .returning({ id: healthCheckAutoIncidents.id });
+            if (updated.length > 0) {
+              logger.debug(
+                `Marked auto-incident mapping closed for resolved incident ${incidentId}`,
+              );
+            }
+          },
+          { mode: "work-queue", workerGroup: "auto-incident-sync" },
+        );
+
         logger.debug("✅ Health Check Backend afterPluginsReady complete.");
       },
     });

package/src/notification-defaults-config.ts

@@ -0,0 +1,10 @@
+/**
+ * Versioned schema used by `ConfigService` to persist platform-wide
+ * notification defaults. The shape is the runtime `NotificationPolicy`
+ * itself — each field has a built-in compile-time default so an empty
+ * stored value still parses to a valid policy.
+ */
+export { NotificationPolicySchema as notificationDefaultsConfigV1 } from "@checkstack/healthcheck-common";
+
+export const NOTIFICATION_DEFAULTS_CONFIG_ID = "healthcheck.notification-defaults";
+export const NOTIFICATION_DEFAULTS_CONFIG_VERSION = 1;