@checkstack/healthcheck-backend 1.2.0 → 1.4.0

package/src/auto-incident-close-job.ts

@@ -1,164 +0,0 @@
-import { and, eq, gte, isNotNull, isNull } from "drizzle-orm";
-import type { Logger, SafeDatabase } from "@checkstack/backend-api";
-import type { InferClient } from "@checkstack/common";
-import { IncidentApi } from "@checkstack/incident-common";
-import type { QueueManager } from "@checkstack/queue-api";
-import * as schema from "./schema";
-import { healthCheckAutoIncidents, healthCheckRuns } from "./schema";
-
-type Db = SafeDatabase<typeof schema>;
-type IncidentClient = InferClient<typeof IncidentApi>;
-
-const AUTO_CLOSE_QUEUE = "health-check-auto-incident-close";
-
-interface AutoCloseJobPayload {
-  trigger: "scheduled";
-}
-
-interface AutoCloseJobDeps {
-  db: Db;
-  logger: Logger;
-  queueManager: QueueManager;
-  incidentClient: IncidentClient;
-  /**
-   * How often the worker ticks. Default 60s. Set lower in tests.
-   */
-  intervalSeconds?: number;
-}
-
-const DEFAULT_INTERVAL_SECONDS = 60;
-
-/**
- * Background worker that resolves auto-opened incidents once the
- * underlying system has stayed healthy for the per-incident cooldown.
- * The cooldown is snapshot per-row at open time (see
- * `healthCheckAutoIncidents.cooldownMinutes`) so a policy change does
- * not retroactively alter the close behaviour of incidents already in
- * flight. A `null` cooldown means "never auto-close" — the worker
- * skips those rows and an operator must resolve them manually.
- */
-export async function setupAutoIncidentCloseJob(deps: AutoCloseJobDeps) {
-  const {
-    queueManager,
-    logger,
-    db,
-    incidentClient,
-    intervalSeconds = DEFAULT_INTERVAL_SECONDS,
-  } = deps;
-
-  const queue = queueManager.getQueue<AutoCloseJobPayload>(AUTO_CLOSE_QUEUE);
-
-  await queue.consume(
-    async () => {
-      await runAutoIncidentCloseJob({ db, logger, incidentClient });
-    },
-    { consumerGroup: "auto-incident-close-worker" },
-  );
-
-  await queue.scheduleRecurring(
-    { trigger: "scheduled" },
-    {
-      jobId: "health-check-auto-incident-close",
-      intervalSeconds,
-    },
-  );
-
-  logger.info(
-    `Health check auto-incident close job scheduled (interval ${intervalSeconds}s; cooldown is per-incident)`,
-  );
-}
-
-/**
- * Resolve any open auto-incidents whose linked system has been
- * steadily healthy for at least their snapshot `cooldownMinutes`. Rows
- * with a null cooldown are skipped. Each incident is processed
- * independently; one failure does not abort the sweep.
- */
-export async function runAutoIncidentCloseJob({
-  db,
-  logger,
-  incidentClient,
-}: {
-  db: Db;
-  logger: Logger;
-  incidentClient: IncidentClient;
-}): Promise<{ closed: number }> {
-  const now = new Date();
-
-  // All open auto-incidents with a non-null cooldown — rows with null
-  // cooldown opted out of auto-close entirely.
-  const open = await db
-    .select({
-      id: healthCheckAutoIncidents.id,
-      incidentId: healthCheckAutoIncidents.incidentId,
-      systemId: healthCheckAutoIncidents.systemId,
-      openedAt: healthCheckAutoIncidents.openedAt,
-      cooldownMinutes: healthCheckAutoIncidents.cooldownMinutes,
-    })
-    .from(healthCheckAutoIncidents)
-    .where(
-      and(
-        isNull(healthCheckAutoIncidents.closedAt),
-        isNotNull(healthCheckAutoIncidents.cooldownMinutes),
-      ),
-    );
-
-  let closed = 0;
-
-  for (const row of open) {
-    try {
-      const cooldownMinutes = row.cooldownMinutes;
-      if (cooldownMinutes === null) continue; // narrows the type
-
-      const cooldownStart = new Date(now.getTime() - cooldownMinutes * 60_000);
-
-      // Require the cooldown to have elapsed since the incident was
-      // opened in the first place. Without this, a system that was
-      // healthy *before* we opened the incident would be auto-closed on
-      // the very first tick.
-      if (row.openedAt > cooldownStart) {
-        continue;
-      }
-
-      // Has the system had any unhealthy runs inside the cooldown?
-      const recentUnhealthy = await db
-        .select({ id: healthCheckRuns.id })
-        .from(healthCheckRuns)
-        .where(
-          and(
-            eq(healthCheckRuns.systemId, row.systemId),
-            eq(healthCheckRuns.status, "unhealthy"),
-            gte(healthCheckRuns.timestamp, cooldownStart),
-          ),
-        )
-        .limit(1);
-
-      if (recentUnhealthy.length > 0) {
-        continue;
-      }
-
-      // Steady-state healthy → resolve.
-      await incidentClient.resolveAutoIncident({
-        id: row.incidentId,
-        message: `Auto-resolved: system stayed healthy for ${cooldownMinutes} minutes.`,
-      });
-
-      await db
-        .update(healthCheckAutoIncidents)
-        .set({ closedAt: new Date() })
-        .where(eq(healthCheckAutoIncidents.id, row.id));
-
-      closed += 1;
-      logger.info(
-        `Auto-closed incident ${row.incidentId} for system ${row.systemId}`,
-      );
-    } catch (error) {
-      logger.warn(
-        `Auto-close failed for incident ${row.incidentId} (system ${row.systemId}):`,
-        error,
-      );
-    }
-  }
-
-  return { closed };
-}

package/src/auto-incident.test.ts

@@ -1,196 +0,0 @@
-import { describe, it, expect } from "bun:test";
-import type {
-  HealthCheckStatus,
-  NotificationPolicy,
-} from "@checkstack/healthcheck-common";
-import {
-  isTransitionToUnhealthy,
-  shouldOpenForFlapping,
-  shouldOpenForSustainedUnhealthy,
-} from "./auto-incident";
-
-const ALL_STATES: HealthCheckStatus[] = ["healthy", "degraded", "unhealthy"];
-
-function policy(overrides: Partial<NotificationPolicy> = {}): NotificationPolicy {
-  return {
-    suppressDeEscalations: false,
-    autoOpenIncidentOnUnhealthy: true,
-    useNotificationSuppression: true,
-    skipDuringMaintenance: true,
-    sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 30 },
-    flappingTrigger: { enabled: true, transitions: 3, windowMinutes: 60 },
-    autoCloseAfterMinutes: 30,
-    ...overrides,
-  };
-}
-
-describe("isTransitionToUnhealthy", () => {
-  it("returns true on healthy → unhealthy", () => {
-    expect(isTransitionToUnhealthy("healthy", "unhealthy")).toBe(true);
-  });
-
-  it("returns true on degraded → unhealthy", () => {
-    expect(isTransitionToUnhealthy("degraded", "unhealthy")).toBe(true);
-  });
-
-  it("returns true on undefined → unhealthy (first-ever evaluation)", () => {
-    expect(isTransitionToUnhealthy(undefined, "unhealthy")).toBe(true);
-  });
-
-  it("returns false when staying unhealthy", () => {
-    expect(isTransitionToUnhealthy("unhealthy", "unhealthy")).toBe(false);
-  });
-
-  for (const next of ALL_STATES) {
-    if (next === "unhealthy") continue;
-    it(`returns false when transitioning to ${next}`, () => {
-      for (const prev of [...ALL_STATES, undefined]) {
-        expect(isTransitionToUnhealthy(prev, next)).toBe(false);
-      }
-    });
-  }
-});
-
-describe("shouldOpenForFlapping", () => {
-  it("never opens when auto-open is disabled at the top level", () => {
-    const p = policy({ autoOpenIncidentOnUnhealthy: false });
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
-    ).toBe(false);
-  });
-
-  it("never opens when the flapping trigger itself is disabled", () => {
-    const p = policy({
-      flappingTrigger: { enabled: false, transitions: 1, windowMinutes: 60 },
-    });
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
-    ).toBe(false);
-  });
-
-  it("does not open below the configured transition count", () => {
-    const p = policy(); // default transitions: 3
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 1 }),
-    ).toBe(false);
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 2 }),
-    ).toBe(false);
-  });
-
-  it("opens once the count reaches the threshold", () => {
-    const p = policy();
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 3 }),
-    ).toBe(true);
-  });
-
-  it("stays open above the threshold (no upper bound)", () => {
-    const p = policy();
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 99 }),
-    ).toBe(true);
-  });
-});
-
-describe("shouldOpenForSustainedUnhealthy", () => {
-  it("never opens when auto-open is disabled at the top level", () => {
-    const p = policy({ autoOpenIncidentOnUnhealthy: false });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 10 * 60 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("never opens when the sustained trigger itself is disabled", () => {
-    const p = policy({
-      sustainedUnhealthyTrigger: { enabled: false, durationMinutes: 30 },
-    });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 10 * 60 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("does not open below the configured duration", () => {
-    // 29 minutes < 30 minute threshold
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 29 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("opens exactly at the threshold", () => {
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 30 * 60_000,
-      }),
-    ).toBe(true);
-  });
-
-  it("opens beyond the threshold", () => {
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 60 * 60_000,
-      }),
-    ).toBe(true);
-  });
-
-  it("respects a custom duration", () => {
-    const p = policy({
-      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 5 },
-    });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 4 * 60_000,
-      }),
-    ).toBe(false);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 5 * 60_000,
-      }),
-    ).toBe(true);
-  });
-});
-
-describe("flapping vs sustained", () => {
-  // The two triggers cover different failure modes. Both should fire
-  // on their respective inputs; either is sufficient to open.
-  it("flapping fires on persistent flapping where each phase is brief", () => {
-    // Check has flapped 3 times in the last hour but each unhealthy
-    // phase was only 5 min long (so sustained would never fire).
-    expect(
-      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 3 }),
-    ).toBe(true);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 5 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("sustained fires on a real outage that hasn't flapped yet", () => {
-    // Only 1 transition (the original break), but it has been
-    // unhealthy for 45 minutes straight.
-    expect(
-      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 1 }),
-    ).toBe(false);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 45 * 60_000,
-      }),
-    ).toBe(true);
-  });
-});

package/src/auto-incident.ts

@@ -1,332 +0,0 @@
-import { and, desc, eq, gte, isNotNull, isNull, sql } from "drizzle-orm";
-import type {
-  HealthCheckStatus,
-  NotificationPolicy,
-} from "@checkstack/healthcheck-common";
-import type { Logger, SafeDatabase } from "@checkstack/backend-api";
-import type { InferClient } from "@checkstack/common";
-import { IncidentApi } from "@checkstack/incident-common";
-import { MaintenanceApi } from "@checkstack/maintenance-common";
-import {
-  healthCheckAutoIncidents,
-  healthCheckRuns,
-  healthCheckUnhealthyTransitions,
-} from "./schema";
-import * as schema from "./schema";
-
-type Db = SafeDatabase<typeof schema>;
-type IncidentClient = InferClient<typeof IncidentApi>;
-type MaintenanceClient = InferClient<typeof MaintenanceApi>;
-
-/**
- * Returns true when the per-check evaluated state went from anything
- * other than `unhealthy` to `unhealthy` between two evaluations.
- */
-export function isTransitionToUnhealthy(
-  previous: HealthCheckStatus | undefined,
-  next: HealthCheckStatus,
-): boolean {
-  return next === "unhealthy" && previous !== "unhealthy";
-}
-
-/**
- * Record a transition-to-unhealthy in the audit table and return the
- * total transition count for this check inside the configured window
- * (the new row is included in the count). When `since` is provided,
- * only transitions strictly after that timestamp are counted — used
- * to ensure a freshly-opened auto-incident isn't re-triggered by
- * pre-close transitions after the prior incident was resolved.
- */
-export async function recordUnhealthyTransition({
-  db,
-  configurationId,
-  systemId,
-  windowMinutes,
-  since,
-  now = new Date(),
-}: {
-  db: Db;
-  configurationId: string;
-  systemId: string;
-  windowMinutes: number;
-  since?: Date;
-  now?: Date;
-}): Promise<number> {
-  await db.insert(healthCheckUnhealthyTransitions).values({
-    configurationId,
-    systemId,
-    transitionedAt: now,
-  });
-
-  const windowStart = new Date(now.getTime() - windowMinutes * 60_000);
-  const lowerBound =
-    since && since > windowStart ? since : windowStart;
-
-  const result = await db
-    .select({ count: sql<number>`COUNT(*)::int` })
-    .from(healthCheckUnhealthyTransitions)
-    .where(
-      and(
-        eq(healthCheckUnhealthyTransitions.configurationId, configurationId),
-        eq(healthCheckUnhealthyTransitions.systemId, systemId),
-        gte(healthCheckUnhealthyTransitions.transitionedAt, lowerBound),
-      ),
-    );
-
-  return result[0]?.count ?? 0;
-}
-
-/**
- * Decide whether the flapping trigger should open an auto-incident.
- * Returns false when the trigger is disabled or the count is below
- * the configured threshold.
- */
-export function shouldOpenForFlapping({
-  policy,
-  recentTransitionCount,
-}: {
-  policy: NotificationPolicy;
-  recentTransitionCount: number;
-}): boolean {
-  if (!policy.autoOpenIncidentOnUnhealthy) return false;
-  if (!policy.flappingTrigger.enabled) return false;
-  return recentTransitionCount >= policy.flappingTrigger.transitions;
-}
-
-/**
- * Decide whether the sustained-duration trigger should open an
- * auto-incident given the elapsed-unhealthy time for this check.
- */
-export function shouldOpenForSustainedUnhealthy({
-  policy,
-  unhealthyForMs,
-}: {
-  policy: NotificationPolicy;
-  /** How long the check has been continuously unhealthy. */
-  unhealthyForMs: number;
-}): boolean {
-  if (!policy.autoOpenIncidentOnUnhealthy) return false;
-  if (!policy.sustainedUnhealthyTrigger.enabled) return false;
-  const thresholdMs =
-    policy.sustainedUnhealthyTrigger.durationMinutes * 60_000;
-  return unhealthyForMs >= thresholdMs;
-}
-
-/**
- * Find the most recent transition to `unhealthy` for this check that
- * happened after `since` (if provided). Used by the sustained-trigger
- * evaluator to compute "how long has the check been unhealthy?"
- */
-export async function findUnhealthySince({
-  db,
-  configurationId,
-  systemId,
-  since,
-}: {
-  db: Db;
-  configurationId: string;
-  systemId: string;
-  since?: Date;
-}): Promise<Date | undefined> {
-  const conditions = [
-    eq(healthCheckUnhealthyTransitions.configurationId, configurationId),
-    eq(healthCheckUnhealthyTransitions.systemId, systemId),
-  ];
-  if (since) {
-    conditions.push(gte(healthCheckUnhealthyTransitions.transitionedAt, since));
-  }
-
-  const [row] = await db
-    .select({
-      transitionedAt: healthCheckUnhealthyTransitions.transitionedAt,
-    })
-    .from(healthCheckUnhealthyTransitions)
-    .where(and(...conditions))
-    .orderBy(desc(healthCheckUnhealthyTransitions.transitionedAt))
-    .limit(1);
-
-  return row?.transitionedAt;
-}
-
-/**
- * Find any currently-active (closedAt IS NULL) auto-incident for the
- * system. Used to avoid opening a duplicate when one is already open.
- */
-export async function findActiveAutoIncident({
-  db,
-  systemId,
-}: {
-  db: Db;
-  systemId: string;
-}): Promise<{ id: string; incidentId: string } | undefined> {
-  const rows = await db
-    .select({
-      id: healthCheckAutoIncidents.id,
-      incidentId: healthCheckAutoIncidents.incidentId,
-    })
-    .from(healthCheckAutoIncidents)
-    .where(
-      and(
-        eq(healthCheckAutoIncidents.systemId, systemId),
-        isNull(healthCheckAutoIncidents.closedAt),
-      ),
-    )
-    .limit(1);
-
-  return rows[0];
-}
-
-/**
- * Most recent close time for an auto-incident on this assignment, or
- * undefined if none has ever closed. Used to gate re-opens behind a
- * "must recover first" rule.
- */
-export async function findLastAutoIncidentClose({
-  db,
-  systemId,
-  configurationId,
-}: {
-  db: Db;
-  systemId: string;
-  configurationId: string;
-}): Promise<Date | undefined> {
-  const [row] = await db
-    .select({ closedAt: healthCheckAutoIncidents.closedAt })
-    .from(healthCheckAutoIncidents)
-    .where(
-      and(
-        eq(healthCheckAutoIncidents.systemId, systemId),
-        eq(healthCheckAutoIncidents.configurationId, configurationId),
-        isNotNull(healthCheckAutoIncidents.closedAt),
-      ),
-    )
-    .orderBy(desc(healthCheckAutoIncidents.closedAt))
-    .limit(1);
-
-  return row?.closedAt ?? undefined;
-}
-
-/**
- * Has this check produced at least one healthy run since the given
- * timestamp? Used to confirm the system has actually recovered between
- * the last auto-incident close and now before a new auto-incident is
- * allowed to open.
- */
-export async function hasHealthyRunSince({
-  db,
-  systemId,
-  configurationId,
-  since,
-}: {
-  db: Db;
-  systemId: string;
-  configurationId: string;
-  since: Date;
-}): Promise<boolean> {
-  const [row] = await db
-    .select({ id: healthCheckRuns.id })
-    .from(healthCheckRuns)
-    .where(
-      and(
-        eq(healthCheckRuns.systemId, systemId),
-        eq(healthCheckRuns.configurationId, configurationId),
-        eq(healthCheckRuns.status, "healthy"),
-        gte(healthCheckRuns.timestamp, since),
-      ),
-    )
-    .limit(1);
-
-  return !!row;
-}
-
-/**
- * Check whether the system currently has an active maintenance window
- * with suppression. Falls back to "not suppressed" on errors so a
- * downstream outage doesn't accidentally block legitimate incidents.
- */
-export async function isMaintenanceSuppressed({
-  maintenanceClient,
-  systemId,
-  logger,
-}: {
-  maintenanceClient: MaintenanceClient;
-  systemId: string;
-  logger: Logger;
-}): Promise<boolean> {
-  try {
-    const { suppressed } =
-      await maintenanceClient.hasActiveMaintenanceWithSuppression({ systemId });
-    return suppressed;
-  } catch (error) {
-    logger.warn(
-      `Failed to check maintenance for ${systemId} during auto-incident decision; assuming not suppressed:`,
-      error,
-    );
-    return false;
-  }
-}
-
-/**
- * Open an auto-incident through the incident plugin's service-level
- * RPC and persist the mapping so the auto-close worker can find and
- * resolve it later. No-op (returns existing mapping) when an active
- * auto-incident already exists for the system.
- */
-export async function openAutoIncident({
-  db,
-  incidentClient,
-  logger,
-  systemId,
-  systemName,
-  configurationId,
-  configurationName,
-  policy,
-  reason,
-}: {
-  db: Db;
-  incidentClient: IncidentClient;
-  logger: Logger;
-  systemId: string;
-  systemName: string;
-  configurationId: string;
-  configurationName: string;
-  policy: NotificationPolicy;
-  /** Short human-readable phrase for the incident description. */
-  reason: string;
-}): Promise<{ incidentId: string } | undefined> {
-  const existing = await findActiveAutoIncident({ db, systemId });
-  if (existing) {
-    return { incidentId: existing.incidentId };
-  }
-
-  try {
-    const { id: incidentId } = await incidentClient.createAutoIncident({
-      title: `${systemName} is critical`,
-      description: `Auto-opened by health check **${configurationName}** (${reason}).`,
-      severity: "critical",
-      suppressNotifications: policy.useNotificationSuppression,
-      systemIds: [systemId],
-      initialMessage: `Health check \`${configurationName}\` triggered the auto-incident: ${reason}.`,
-    });
-
-    await db.insert(healthCheckAutoIncidents).values({
-      incidentId,
-      systemId,
-      configurationId,
-      cooldownMinutes: policy.autoCloseAfterMinutes,
-    });
-
-    logger.info(
-      `Auto-opened incident ${incidentId} for system ${systemId} (check ${configurationId}; ${reason})`,
-    );
-    return { incidentId };
-  } catch (error) {
-    // Auto-incident creation is best-effort — failure here shouldn't
-    // block the rest of the health-check flow.
-    logger.warn(
-      `Failed to open auto-incident for system ${systemId} (check ${configurationId}):`,
-      error,
-    );
-    return undefined;
-  }
-}