@checkstack/healthcheck-backend 1.3.0 → 1.4.0

package/src/state-transitions.test.ts

@@ -0,0 +1,126 @@
+import { describe, it, expect, mock } from "bun:test";
+import {
+  countStateTransitionsInWindow,
+  findInStatusSince,
+  recordStateTransition,
+} from "./state-transitions";
+
+/**
+ * Minimal fluent mock for `db.select(...).from(...).where(...).orderBy(...).limit(...)`
+ * that resolves to the provided rows.
+ */
+function selectMockDb(rows: Array<{ transitionedAt: Date }>) {
+  return {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() => ({
+          orderBy: mock(() => ({
+            limit: mock(() => Promise.resolve(rows)),
+          })),
+        })),
+      })),
+    })),
+  };
+}
+
+describe("findInStatusSince", () => {
+  it("returns the most-recent transitionedAt for the status", async () => {
+    const since = new Date("2026-05-30T10:00:00.000Z");
+    const db = selectMockDb([{ transitionedAt: since }]);
+    const result = await findInStatusSince({
+      db: db as never,
+      systemId: "system-1",
+      status: "unhealthy",
+    });
+    expect(result).toBe(since);
+  });
+
+  it("returns null (fail-safe) when no transition row exists", async () => {
+    const db = selectMockDb([]);
+    const result = await findInStatusSince({
+      db: db as never,
+      systemId: "system-1",
+      status: "degraded",
+    });
+    expect(result).toBeNull();
+  });
+});
+
+describe("recordStateTransition", () => {
+  it("inserts a row with from/to status and the provided timestamp", async () => {
+    const values =
+      mock<(v: Record<string, unknown>) => Promise<void>>(() =>
+        Promise.resolve(),
+      );
+    const db = { insert: mock(() => ({ values })) };
+    const now = new Date("2026-05-30T12:00:00.000Z");
+
+    await recordStateTransition({
+      db: db as never,
+      systemId: "system-1",
+      configurationId: "config-1",
+      fromStatus: "healthy",
+      toStatus: "unhealthy",
+      now,
+    });
+
+    expect(values).toHaveBeenCalledTimes(1);
+    expect(values.mock.calls[0]?.[0]).toEqual({
+      systemId: "system-1",
+      configurationId: "config-1",
+      fromStatus: "healthy",
+      toStatus: "unhealthy",
+      transitionedAt: now,
+    });
+  });
+
+  it("stores null fromStatus on the first-ever transition", async () => {
+    const values =
+      mock<(v: Record<string, unknown>) => Promise<void>>(() =>
+        Promise.resolve(),
+      );
+    const db = { insert: mock(() => ({ values })) };
+
+    await recordStateTransition({
+      db: db as never,
+      systemId: "system-1",
+      configurationId: "config-1",
+      fromStatus: undefined,
+      toStatus: "degraded",
+    });
+
+    const arg = values.mock.calls[0]?.[0] as { fromStatus: unknown };
+    expect(arg.fromStatus).toBeNull();
+  });
+});
+
+describe("countStateTransitionsInWindow", () => {
+  /** Mock for `db.select({count}).from(...).where(...)` resolving to [{count}]. */
+  function countMockDb(count: number) {
+    const where = mock(() => Promise.resolve([{ count }]));
+    const from = mock(() => ({ where }));
+    const select = mock(() => ({ from }));
+    return { db: { select }, where };
+  }
+
+  it("returns the windowed count", async () => {
+    const { db } = countMockDb(4);
+    const result = await countStateTransitionsInWindow({
+      db: db as never,
+      systemId: "system-1",
+      windowMinutes: 60,
+    });
+    expect(result).toBe(4);
+  });
+
+  it("returns 0 (fail-safe) when the query yields no rows", async () => {
+    const where = mock(() => Promise.resolve([]));
+    const db = { select: mock(() => ({ from: mock(() => ({ where })) })) };
+    const result = await countStateTransitionsInWindow({
+      db: db as never,
+      systemId: "system-1",
+      windowMinutes: 30,
+    });
+    expect(result).toBe(0);
+  });
+});

package/src/state-transitions.ts

@@ -0,0 +1,112 @@
+import { and, desc, eq, gte, sql } from "drizzle-orm";
+import type { HealthCheckStatus } from "@checkstack/healthcheck-common";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import { healthCheckStateTransitions } from "./schema";
+import * as schema from "./schema";
+
+type Db = SafeDatabase<typeof schema>;
+
+/**
+ * Record an aggregate health-status transition for a system. Called at
+ * the same point `systemHealthChanged` fires (one row per aggregate
+ * transition, which is rare). `fromStatus` is null on the first-ever
+ * recorded transition for a system.
+ */
+export async function recordStateTransition({
+  db,
+  systemId,
+  configurationId,
+  fromStatus,
+  toStatus,
+  now = new Date(),
+}: {
+  db: Db;
+  systemId: string;
+  configurationId: string;
+  fromStatus: HealthCheckStatus | undefined;
+  toStatus: HealthCheckStatus;
+  now?: Date;
+}): Promise<void> {
+  await db.insert(healthCheckStateTransitions).values({
+    systemId,
+    configurationId,
+    fromStatus: fromStatus ?? null,
+    toStatus,
+    transitionedAt: now,
+  });
+}
+
+/**
+ * Find the timestamp at which the system most recently entered the
+ * given status (the start of its current streak in that status).
+ *
+ * Fail-safe: when no transition row exists (e.g. the table was pruned
+ * before this system ever transitioned, or it has never changed status)
+ * this returns `null` rather than throwing, so callers degrade to
+ * `inStatusSince: null` instead of failing the whole evaluation.
+ */
+export async function findInStatusSince({
+  db,
+  systemId,
+  status,
+}: {
+  db: Db;
+  systemId: string;
+  status: HealthCheckStatus;
+}): Promise<Date | null> {
+  const [row] = await db
+    .select({ transitionedAt: healthCheckStateTransitions.transitionedAt })
+    .from(healthCheckStateTransitions)
+    .where(
+      and(
+        eq(healthCheckStateTransitions.systemId, systemId),
+        eq(healthCheckStateTransitions.toStatus, status),
+      ),
+    )
+    .orderBy(desc(healthCheckStateTransitions.transitionedAt))
+    .limit(1);
+
+  return row?.transitionedAt ?? null;
+}
+
+/**
+ * Count aggregate state transitions for a system within the trailing
+ * window `[now - windowMinutes, now]`. Generalizes the flapping detector's
+ * "N transitions in M minutes" count beyond the unhealthy-only table.
+ *
+ * When `toStatus` is given, counts only transitions INTO that status
+ * (e.g. flapping = repeated transitions into `unhealthy`); omit it to
+ * count all status changes in the window.
+ *
+ * Fail-safe: returns 0 on any error rather than throwing, so a count
+ * read never wedges an evaluation.
+ */
+export async function countStateTransitionsInWindow({
+  db,
+  systemId,
+  windowMinutes,
+  toStatus,
+  now = new Date(),
+}: {
+  db: Db;
+  systemId: string;
+  windowMinutes: number;
+  toStatus?: HealthCheckStatus;
+  now?: Date;
+}): Promise<number> {
+  const windowStart = new Date(now.getTime() - windowMinutes * 60_000);
+  const conditions = [
+    eq(healthCheckStateTransitions.systemId, systemId),
+    gte(healthCheckStateTransitions.transitionedAt, windowStart),
+  ];
+  if (toStatus) {
+    conditions.push(eq(healthCheckStateTransitions.toStatus, toStatus));
+  }
+
+  const [row] = await db
+    .select({ count: sql<number>`COUNT(*)::int` })
+    .from(healthCheckStateTransitions)
+    .where(and(...conditions));
+
+  return row?.count ?? 0;
+}

package/tsconfig.json

@@ -58,6 +58,15 @@
     {
       "path": "../satellite-backend"
     },
+    {
+      "path": "../script-packages-backend"
+    },
+    {
+      "path": "../secrets-backend"
+    },
+    {
+      "path": "../secrets-common"
+    },
     {
       "path": "../signal-common"
     },

package/src/auto-incident-close-job.ts

@@ -1,164 +0,0 @@
-import { and, eq, gte, isNotNull, isNull } from "drizzle-orm";
-import type { Logger, SafeDatabase } from "@checkstack/backend-api";
-import type { InferClient } from "@checkstack/common";
-import { IncidentApi } from "@checkstack/incident-common";
-import type { QueueManager } from "@checkstack/queue-api";
-import * as schema from "./schema";
-import { healthCheckAutoIncidents, healthCheckRuns } from "./schema";
-
-type Db = SafeDatabase<typeof schema>;
-type IncidentClient = InferClient<typeof IncidentApi>;
-
-const AUTO_CLOSE_QUEUE = "health-check-auto-incident-close";
-
-interface AutoCloseJobPayload {
-  trigger: "scheduled";
-}
-
-interface AutoCloseJobDeps {
-  db: Db;
-  logger: Logger;
-  queueManager: QueueManager;
-  incidentClient: IncidentClient;
-  /**
-   * How often the worker ticks. Default 60s. Set lower in tests.
-   */
-  intervalSeconds?: number;
-}
-
-const DEFAULT_INTERVAL_SECONDS = 60;
-
-/**
- * Background worker that resolves auto-opened incidents once the
- * underlying system has stayed healthy for the per-incident cooldown.
- * The cooldown is snapshot per-row at open time (see
- * `healthCheckAutoIncidents.cooldownMinutes`) so a policy change does
- * not retroactively alter the close behaviour of incidents already in
- * flight. A `null` cooldown means "never auto-close" — the worker
- * skips those rows and an operator must resolve them manually.
- */
-export async function setupAutoIncidentCloseJob(deps: AutoCloseJobDeps) {
-  const {
-    queueManager,
-    logger,
-    db,
-    incidentClient,
-    intervalSeconds = DEFAULT_INTERVAL_SECONDS,
-  } = deps;
-
-  const queue = queueManager.getQueue<AutoCloseJobPayload>(AUTO_CLOSE_QUEUE);
-
-  await queue.consume(
-    async () => {
-      await runAutoIncidentCloseJob({ db, logger, incidentClient });
-    },
-    { consumerGroup: "auto-incident-close-worker" },
-  );
-
-  await queue.scheduleRecurring(
-    { trigger: "scheduled" },
-    {
-      jobId: "health-check-auto-incident-close",
-      intervalSeconds,
-    },
-  );
-
-  logger.info(
-    `Health check auto-incident close job scheduled (interval ${intervalSeconds}s; cooldown is per-incident)`,
-  );
-}
-
-/**
- * Resolve any open auto-incidents whose linked system has been
- * steadily healthy for at least their snapshot `cooldownMinutes`. Rows
- * with a null cooldown are skipped. Each incident is processed
- * independently; one failure does not abort the sweep.
- */
-export async function runAutoIncidentCloseJob({
-  db,
-  logger,
-  incidentClient,
-}: {
-  db: Db;
-  logger: Logger;
-  incidentClient: IncidentClient;
-}): Promise<{ closed: number }> {
-  const now = new Date();
-
-  // All open auto-incidents with a non-null cooldown — rows with null
-  // cooldown opted out of auto-close entirely.
-  const open = await db
-    .select({
-      id: healthCheckAutoIncidents.id,
-      incidentId: healthCheckAutoIncidents.incidentId,
-      systemId: healthCheckAutoIncidents.systemId,
-      openedAt: healthCheckAutoIncidents.openedAt,
-      cooldownMinutes: healthCheckAutoIncidents.cooldownMinutes,
-    })
-    .from(healthCheckAutoIncidents)
-    .where(
-      and(
-        isNull(healthCheckAutoIncidents.closedAt),
-        isNotNull(healthCheckAutoIncidents.cooldownMinutes),
-      ),
-    );
-
-  let closed = 0;
-
-  for (const row of open) {
-    try {
-      const cooldownMinutes = row.cooldownMinutes;
-      if (cooldownMinutes === null) continue; // narrows the type
-
-      const cooldownStart = new Date(now.getTime() - cooldownMinutes * 60_000);
-
-      // Require the cooldown to have elapsed since the incident was
-      // opened in the first place. Without this, a system that was
-      // healthy *before* we opened the incident would be auto-closed on
-      // the very first tick.
-      if (row.openedAt > cooldownStart) {
-        continue;
-      }
-
-      // Has the system had any unhealthy runs inside the cooldown?
-      const recentUnhealthy = await db
-        .select({ id: healthCheckRuns.id })
-        .from(healthCheckRuns)
-        .where(
-          and(
-            eq(healthCheckRuns.systemId, row.systemId),
-            eq(healthCheckRuns.status, "unhealthy"),
-            gte(healthCheckRuns.timestamp, cooldownStart),
-          ),
-        )
-        .limit(1);
-
-      if (recentUnhealthy.length > 0) {
-        continue;
-      }
-
-      // Steady-state healthy → resolve.
-      await incidentClient.resolveAutoIncident({
-        id: row.incidentId,
-        message: `Auto-resolved: system stayed healthy for ${cooldownMinutes} minutes.`,
-      });
-
-      await db
-        .update(healthCheckAutoIncidents)
-        .set({ closedAt: new Date() })
-        .where(eq(healthCheckAutoIncidents.id, row.id));
-
-      closed += 1;
-      logger.info(
-        `Auto-closed incident ${row.incidentId} for system ${row.systemId}`,
-      );
-    } catch (error) {
-      logger.warn(
-        `Auto-close failed for incident ${row.incidentId} (system ${row.systemId}):`,
-        error,
-      );
-    }
-  }
-
-  return { closed };
-}

package/src/auto-incident.test.ts

@@ -1,196 +0,0 @@
-import { describe, it, expect } from "bun:test";
-import type {
-  HealthCheckStatus,
-  NotificationPolicy,
-} from "@checkstack/healthcheck-common";
-import {
-  isTransitionToUnhealthy,
-  shouldOpenForFlapping,
-  shouldOpenForSustainedUnhealthy,
-} from "./auto-incident";
-
-const ALL_STATES: HealthCheckStatus[] = ["healthy", "degraded", "unhealthy"];
-
-function policy(overrides: Partial<NotificationPolicy> = {}): NotificationPolicy {
-  return {
-    suppressDeEscalations: false,
-    autoOpenIncidentOnUnhealthy: true,
-    useNotificationSuppression: true,
-    skipDuringMaintenance: true,
-    sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 30 },
-    flappingTrigger: { enabled: true, transitions: 3, windowMinutes: 60 },
-    autoCloseAfterMinutes: 30,
-    ...overrides,
-  };
-}
-
-describe("isTransitionToUnhealthy", () => {
-  it("returns true on healthy → unhealthy", () => {
-    expect(isTransitionToUnhealthy("healthy", "unhealthy")).toBe(true);
-  });
-
-  it("returns true on degraded → unhealthy", () => {
-    expect(isTransitionToUnhealthy("degraded", "unhealthy")).toBe(true);
-  });
-
-  it("returns true on undefined → unhealthy (first-ever evaluation)", () => {
-    expect(isTransitionToUnhealthy(undefined, "unhealthy")).toBe(true);
-  });
-
-  it("returns false when staying unhealthy", () => {
-    expect(isTransitionToUnhealthy("unhealthy", "unhealthy")).toBe(false);
-  });
-
-  for (const next of ALL_STATES) {
-    if (next === "unhealthy") continue;
-    it(`returns false when transitioning to ${next}`, () => {
-      for (const prev of [...ALL_STATES, undefined]) {
-        expect(isTransitionToUnhealthy(prev, next)).toBe(false);
-      }
-    });
-  }
-});
-
-describe("shouldOpenForFlapping", () => {
-  it("never opens when auto-open is disabled at the top level", () => {
-    const p = policy({ autoOpenIncidentOnUnhealthy: false });
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
-    ).toBe(false);
-  });
-
-  it("never opens when the flapping trigger itself is disabled", () => {
-    const p = policy({
-      flappingTrigger: { enabled: false, transitions: 1, windowMinutes: 60 },
-    });
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 999 }),
-    ).toBe(false);
-  });
-
-  it("does not open below the configured transition count", () => {
-    const p = policy(); // default transitions: 3
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 1 }),
-    ).toBe(false);
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 2 }),
-    ).toBe(false);
-  });
-
-  it("opens once the count reaches the threshold", () => {
-    const p = policy();
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 3 }),
-    ).toBe(true);
-  });
-
-  it("stays open above the threshold (no upper bound)", () => {
-    const p = policy();
-    expect(
-      shouldOpenForFlapping({ policy: p, recentTransitionCount: 99 }),
-    ).toBe(true);
-  });
-});
-
-describe("shouldOpenForSustainedUnhealthy", () => {
-  it("never opens when auto-open is disabled at the top level", () => {
-    const p = policy({ autoOpenIncidentOnUnhealthy: false });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 10 * 60 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("never opens when the sustained trigger itself is disabled", () => {
-    const p = policy({
-      sustainedUnhealthyTrigger: { enabled: false, durationMinutes: 30 },
-    });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 10 * 60 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("does not open below the configured duration", () => {
-    // 29 minutes < 30 minute threshold
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 29 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("opens exactly at the threshold", () => {
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 30 * 60_000,
-      }),
-    ).toBe(true);
-  });
-
-  it("opens beyond the threshold", () => {
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 60 * 60_000,
-      }),
-    ).toBe(true);
-  });
-
-  it("respects a custom duration", () => {
-    const p = policy({
-      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 5 },
-    });
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 4 * 60_000,
-      }),
-    ).toBe(false);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: p,
-        unhealthyForMs: 5 * 60_000,
-      }),
-    ).toBe(true);
-  });
-});
-
-describe("flapping vs sustained", () => {
-  // The two triggers cover different failure modes. Both should fire
-  // on their respective inputs; either is sufficient to open.
-  it("flapping fires on persistent flapping where each phase is brief", () => {
-    // Check has flapped 3 times in the last hour but each unhealthy
-    // phase was only 5 min long (so sustained would never fire).
-    expect(
-      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 3 }),
-    ).toBe(true);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 5 * 60_000,
-      }),
-    ).toBe(false);
-  });
-
-  it("sustained fires on a real outage that hasn't flapped yet", () => {
-    // Only 1 transition (the original break), but it has been
-    // unhealthy for 45 minutes straight.
-    expect(
-      shouldOpenForFlapping({ policy: policy(), recentTransitionCount: 1 }),
-    ).toBe(false);
-    expect(
-      shouldOpenForSustainedUnhealthy({
-        policy: policy(),
-        unhealthyForMs: 45 * 60_000,
-      }),
-    ).toBe(true);
-  });
-});