@checkstack/healthcheck-backend 1.1.3 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,23 +14,24 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/backend-api": "0.15.3",
-    "@checkstack/cache-api": "0.3.2",
-    "@checkstack/cache-utils": "0.2.7",
-    "@checkstack/catalog-backend": "1.1.3",
-    "@checkstack/catalog-common": "2.2.0",
-    "@checkstack/command-backend": "0.1.27",
-    "@checkstack/common": "0.10.0",
-    "@checkstack/gitops-backend": "0.3.3",
-    "@checkstack/gitops-common": "0.4.0",
-    "@checkstack/healthcheck-common": "1.1.0",
-    "@checkstack/incident-common": "1.2.0",
-    "@checkstack/integration-backend": "0.1.27",
-    "@checkstack/maintenance-common": "1.2.0",
-    "@checkstack/notification-common": "1.1.0",
-    "@checkstack/queue-api": "0.3.2",
-    "@checkstack/satellite-backend": "0.3.3",
-    "@checkstack/signal-common": "0.2.3",
+    "@checkstack/backend-api": "0.17.0",
+    "@checkstack/cache-api": "0.3.4",
+    "@checkstack/cache-utils": "0.2.9",
+    "@checkstack/catalog-backend": "1.1.5",
+    "@checkstack/catalog-common": "2.2.2",
+    "@checkstack/command-backend": "0.1.29",
+    "@checkstack/common": "0.11.0",
+    "@checkstack/gitops-backend": "0.3.5",
+    "@checkstack/gitops-common": "0.4.1",
+    "@checkstack/healthcheck-common": "1.1.2",
+    "@checkstack/incident-backend": "1.1.5",
+    "@checkstack/incident-common": "1.2.2",
+    "@checkstack/integration-backend": "0.1.29",
+    "@checkstack/maintenance-common": "1.2.2",
+    "@checkstack/notification-common": "1.2.0",
+    "@checkstack/queue-api": "0.3.4",
+    "@checkstack/satellite-backend": "0.3.5",
+    "@checkstack/signal-common": "0.2.4",
     "@hono/zod-validator": "^0.7.6",
     "drizzle-orm": "^0.45.0",
     "hono": "^4.12.14",
@@ -40,8 +41,8 @@
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.5",
-    "@checkstack/scripts": "0.3.2",
-    "@checkstack/test-utils-backend": "0.1.27",
+    "@checkstack/scripts": "0.3.3",
+    "@checkstack/test-utils-backend": "0.1.29",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/tdigest": "^0.1.5",

package/src/auto-incident-close-job.ts

@@ -0,0 +1,164 @@
+import { and, eq, gte, isNotNull, isNull } from "drizzle-orm";
+import type { Logger, SafeDatabase } from "@checkstack/backend-api";
+import type { InferClient } from "@checkstack/common";
+import { IncidentApi } from "@checkstack/incident-common";
+import type { QueueManager } from "@checkstack/queue-api";
+import * as schema from "./schema";
+import { healthCheckAutoIncidents, healthCheckRuns } from "./schema";
+
+type Db = SafeDatabase<typeof schema>;
+type IncidentClient = InferClient<typeof IncidentApi>;
+
+const AUTO_CLOSE_QUEUE = "health-check-auto-incident-close";
+
+interface AutoCloseJobPayload {
+  trigger: "scheduled";
+}
+
+interface AutoCloseJobDeps {
+  db: Db;
+  logger: Logger;
+  queueManager: QueueManager;
+  incidentClient: IncidentClient;
+  /**
+   * How often the worker ticks. Default 60s. Set lower in tests.
+   */
+  intervalSeconds?: number;
+}
+
+const DEFAULT_INTERVAL_SECONDS = 60;
+
+/**
+ * Background worker that resolves auto-opened incidents once the
+ * underlying system has stayed healthy for the per-incident cooldown.
+ * The cooldown is snapshot per-row at open time (see
+ * `healthCheckAutoIncidents.cooldownMinutes`) so a policy change does
+ * not retroactively alter the close behaviour of incidents already in
+ * flight. A `null` cooldown means "never auto-close" — the worker
+ * skips those rows and an operator must resolve them manually.
+ */
+export async function setupAutoIncidentCloseJob(deps: AutoCloseJobDeps) {
+  const {
+    queueManager,
+    logger,
+    db,
+    incidentClient,
+    intervalSeconds = DEFAULT_INTERVAL_SECONDS,
+  } = deps;
+
+  const queue = queueManager.getQueue<AutoCloseJobPayload>(AUTO_CLOSE_QUEUE);
+
+  await queue.consume(
+    async () => {
+      await runAutoIncidentCloseJob({ db, logger, incidentClient });
+    },
+    { consumerGroup: "auto-incident-close-worker" },
+  );
+
+  await queue.scheduleRecurring(
+    { trigger: "scheduled" },
+    {
+      jobId: "health-check-auto-incident-close",
+      intervalSeconds,
+    },
+  );
+
+  logger.info(
+    `Health check auto-incident close job scheduled (interval ${intervalSeconds}s; cooldown is per-incident)`,
+  );
+}
+
+/**
+ * Resolve any open auto-incidents whose linked system has been
+ * steadily healthy for at least their snapshot `cooldownMinutes`. Rows
+ * with a null cooldown are skipped. Each incident is processed
+ * independently; one failure does not abort the sweep.
+ */
+export async function runAutoIncidentCloseJob({
+  db,
+  logger,
+  incidentClient,
+}: {
+  db: Db;
+  logger: Logger;
+  incidentClient: IncidentClient;
+}): Promise<{ closed: number }> {
+  const now = new Date();
+
+  // All open auto-incidents with a non-null cooldown — rows with null
+  // cooldown opted out of auto-close entirely.
+  const open = await db
+    .select({
+      id: healthCheckAutoIncidents.id,
+      incidentId: healthCheckAutoIncidents.incidentId,
+      systemId: healthCheckAutoIncidents.systemId,
+      openedAt: healthCheckAutoIncidents.openedAt,
+      cooldownMinutes: healthCheckAutoIncidents.cooldownMinutes,
+    })
+    .from(healthCheckAutoIncidents)
+    .where(
+      and(
+        isNull(healthCheckAutoIncidents.closedAt),
+        isNotNull(healthCheckAutoIncidents.cooldownMinutes),
+      ),
+    );
+
+  let closed = 0;
+
+  for (const row of open) {
+    try {
+      const cooldownMinutes = row.cooldownMinutes;
+      if (cooldownMinutes === null) continue; // narrows the type
+
+      const cooldownStart = new Date(now.getTime() - cooldownMinutes * 60_000);
+
+      // Require the cooldown to have elapsed since the incident was
+      // opened in the first place. Without this, a system that was
+      // healthy *before* we opened the incident would be auto-closed on
+      // the very first tick.
+      if (row.openedAt > cooldownStart) {
+        continue;
+      }
+
+      // Has the system had any unhealthy runs inside the cooldown?
+      const recentUnhealthy = await db
+        .select({ id: healthCheckRuns.id })
+        .from(healthCheckRuns)
+        .where(
+          and(
+            eq(healthCheckRuns.systemId, row.systemId),
+            eq(healthCheckRuns.status, "unhealthy"),
+            gte(healthCheckRuns.timestamp, cooldownStart),
+          ),
+        )
+        .limit(1);
+
+      if (recentUnhealthy.length > 0) {
+        continue;
+      }
+
+      // Steady-state healthy → resolve.
+      await incidentClient.resolveAutoIncident({
+        id: row.incidentId,
+        message: `Auto-resolved: system stayed healthy for ${cooldownMinutes} minutes.`,
+      });
+
+      await db
+        .update(healthCheckAutoIncidents)
+        .set({ closedAt: new Date() })
+        .where(eq(healthCheckAutoIncidents.id, row.id));
+
+      closed += 1;
+      logger.info(
+        `Auto-closed incident ${row.incidentId} for system ${row.systemId}`,
+      );
+    } catch (error) {
+      logger.warn(
+        `Auto-close failed for incident ${row.incidentId} (system ${row.systemId}):`,
+        error,
+      );
+    }
+  }
+
+  return { closed };
+}

package/src/auto-incident.test.ts

@@ -0,0 +1,196 @@
+import { describe, it, expect } from "bun:test";
+import type {
+  HealthCheckStatus,
+  NotificationPolicy,
+} from "@checkstack/healthcheck-common";
+import {
+  isTransitionToUnhealthy,
+  shouldOpenForFlapping,
+  shouldOpenForSustainedUnhealthy,
+} from "./auto-incident";
+
+const ALL_STATES: HealthCheckStatus[] = ["healthy", "degraded", "unhealthy"];
+
+function policy(overrides: Partial<NotificationPolicy> = {}): NotificationPolicy {
+  return {
+    suppressDeEscalations: false,
+    autoOpenIncidentOnUnhealthy: true,
+    useNotificationSuppression: true,
+    skipDuringMaintenance: true,
+    sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 30 },
+    flappingTrigger: { enabled: true, transitions: 3, windowMinutes: 60 },
+    autoCloseAfterMinutes: 30,
+    ...overrides,
+  };
+}
+
+describe("isTransitionToUnhealthy", () => {
+  it("returns true on healthy → unhealthy", () => {
+    expect(isTransitionToUnhealthy("healthy", "unhealthy")).toBe(true);
+  });
+
+  it("returns true on degraded → unhealthy", () => {
+    expect(isTransitionToUnhealthy("degraded", "unhealthy")).toBe(true);
+  });
+
+  it("returns true on undefined → unhealthy (first-ever evaluation)", () => {
+    expect(isTransitionToUnhealthy(undefined, "unhealthy")).toBe(true);
+  });
+
+  it("returns false when staying unhealthy", () => {
+    expect(isTransitionToUnhealthy("unhealthy", "unhealthy")).toBe(false);
+  });
+
+  for (const next of ALL_STATES) {
+    if (next === "unhealthy") continue;
+    it(`returns false when transitioning to ${next}`, () => {
+      for (const prev of [...ALL_STATES, undefined]) {
+        expect(isTransitionToUnhealthy(prev, next)).toBe(false);
+      }
+    });
+  }
+});
+
+describe("shouldOpenForFlapping", () => {
+  it("never opens when auto-open is disabled at the top level", () => {
+    const p = policy({ autoOpenIncidentOnUnhealthy: false });
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
+    ).toBe(false);
+  });
+
+  it("never opens when the flapping trigger itself is disabled", () => {
+    const p = policy({
+      flappingTrigger: { enabled: false, transitions: 1, windowMinutes: 60 },
+    });
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
+    ).toBe(false);
+  });
+
+  it("does not open below the configured transition count", () => {
+    const p = policy(); // default transitions: 3
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 1 }),
+    ).toBe(false);
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 2 }),
+    ).toBe(false);
+  });
+
+  it("opens once the count reaches the threshold", () => {
+    const p = policy();
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 3 }),
+    ).toBe(true);
+  });
+
+  it("stays open above the threshold (no upper bound)", () => {
+    const p = policy();
+    expect(
+      shouldOpenForFlapping({ policy: p, recentTransitionCount: 99 }),
+    ).toBe(true);
+  });
+});
+
+describe("shouldOpenForSustainedUnhealthy", () => {
+  it("never opens when auto-open is disabled at the top level", () => {
+    const p = policy({ autoOpenIncidentOnUnhealthy: false });
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: p,
+        unhealthyForMs: 10 * 60 * 60_000,
+      }),
+    ).toBe(false);
+  });
+
+  it("never opens when the sustained trigger itself is disabled", () => {
+    const p = policy({
+      sustainedUnhealthyTrigger: { enabled: false, durationMinutes: 30 },
+    });
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: p,
+        unhealthyForMs: 10 * 60 * 60_000,
+      }),
+    ).toBe(false);
+  });
+
+  it("does not open below the configured duration", () => {
+    // 29 minutes < 30 minute threshold
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: policy(),
+        unhealthyForMs: 29 * 60_000,
+      }),
+    ).toBe(false);
+  });
+
+  it("opens exactly at the threshold", () => {
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: policy(),
+        unhealthyForMs: 30 * 60_000,
+      }),
+    ).toBe(true);
+  });
+
+  it("opens beyond the threshold", () => {
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: policy(),
+        unhealthyForMs: 60 * 60_000,
+      }),
+    ).toBe(true);
+  });
+
+  it("respects a custom duration", () => {
+    const p = policy({
+      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 5 },
+    });
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: p,
+        unhealthyForMs: 4 * 60_000,
+      }),
+    ).toBe(false);
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: p,
+        unhealthyForMs: 5 * 60_000,
+      }),
+    ).toBe(true);
+  });
+});
+
+describe("flapping vs sustained", () => {
+  // The two triggers cover different failure modes. Both should fire
+  // on their respective inputs; either is sufficient to open.
+  it("flapping fires on persistent flapping where each phase is brief", () => {
+    // Check has flapped 3 times in the last hour but each unhealthy
+    // phase was only 5 min long (so sustained would never fire).
+    expect(
+      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 3 }),
+    ).toBe(true);
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: policy(),
+        unhealthyForMs: 5 * 60_000,
+      }),
+    ).toBe(false);
+  });
+
+  it("sustained fires on a real outage that hasn't flapped yet", () => {
+    // Only 1 transition (the original break), but it has been
+    // unhealthy for 45 minutes straight.
+    expect(
+      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 1 }),
+    ).toBe(false);
+    expect(
+      shouldOpenForSustainedUnhealthy({
+        policy: policy(),
+        unhealthyForMs: 45 * 60_000,
+      }),
+    ).toBe(true);
+  });
+});