@checkstack/healthcheck-backend 1.3.0 → 1.4.0

package/src/retention-job.ts

@@ -4,9 +4,10 @@ import {
   healthCheckRuns,
   systemHealthChecks,
   healthCheckAggregates,
+  healthCheckStateTransitions,
   DEFAULT_RETENTION_CONFIG,
 } from "./schema";
-import { eq, and, lt, sql } from "drizzle-orm";
+import { eq, and, lt, sql, desc } from "drizzle-orm";
 import type { QueueManager } from "@checkstack/queue-api";
 
 type Db = SafeDatabase<typeof schema>;
@@ -69,6 +70,29 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
   // Get all unique system-config assignments
   const assignments = await db.select().from(systemHealthChecks);
 
+  // State transitions are system-level (not per-config), so prune them
+  // once per unique system rather than once per assignment. Use the
+  // longest rawRetentionDays among the system's assignments so a single
+  // short-retention check can't drop history another check still wants.
+  const rawRetentionBySystem = new Map<string, number>();
+  for (const assignment of assignments) {
+    const days = (
+      assignment.retentionConfig ?? DEFAULT_RETENTION_CONFIG
+    ).rawRetentionDays;
+    const existing = rawRetentionBySystem.get(assignment.systemId);
+    if (existing === undefined || days > existing) {
+      rawRetentionBySystem.set(assignment.systemId, days);
+    }
+  }
+
+  for (const [systemId, rawRetentionDays] of rawRetentionBySystem) {
+    try {
+      await pruneStateTransitions({ db, systemId, rawRetentionDays });
+    } catch (error) {
+      logger.error(`State-transition prune failed for ${systemId}`, { error });
+    }
+  }
+
   for (const assignment of assignments) {
     const retentionConfig =
       assignment.retentionConfig ?? DEFAULT_RETENTION_CONFIG;
@@ -134,6 +158,46 @@ async function deleteExpiredRawRuns(params: DeleteExpiredRawRunsParams) {
     );
 }
 
+interface PruneStateTransitionsParams {
+  db: Db;
+  systemId: string;
+  rawRetentionDays: number;
+}
+
+/**
+ * Prune aggregate state-transition rows older than the raw-run retention
+ * window, but ALWAYS keep the single most-recent transition for the
+ * system so "in current status since" never blanks for an active streak
+ * (decision D3). Deletes rows that are both older than the cutoff AND
+ * not the newest row.
+ */
+export async function pruneStateTransitions(
+  params: PruneStateTransitionsParams,
+): Promise<void> {
+  const { db, systemId, rawRetentionDays } = params;
+
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
+
+  const [newest] = await db
+    .select({ id: healthCheckStateTransitions.id })
+    .from(healthCheckStateTransitions)
+    .where(eq(healthCheckStateTransitions.systemId, systemId))
+    .orderBy(desc(healthCheckStateTransitions.transitionedAt))
+    .limit(1);
+
+  const conditions = [
+    eq(healthCheckStateTransitions.systemId, systemId),
+    lt(healthCheckStateTransitions.transitionedAt, cutoffDate),
+  ];
+  if (newest) {
+    // Never delete the most-recent row, even if it predates the cutoff.
+    conditions.push(sql`${healthCheckStateTransitions.id} <> ${newest.id}`);
+  }
+
+  await db.delete(healthCheckStateTransitions).where(and(...conditions));
+}
+
 interface RollupParams {
   db: Db;
   systemId: string;

package/src/retention-state-transitions.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, mock } from "bun:test";
+import { pruneStateTransitions } from "./retention-job";
+
+/**
+ * Mock db exposing the two shapes pruneStateTransitions uses:
+ * - select(...).from(...).where(...).orderBy(...).limit(...) -> newest row
+ * - delete(...).where(...) -> prune
+ */
+function makeMockDb(newestRows: Array<{ id: string }>) {
+  const where = mock(() => Promise.resolve());
+  const db = {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() => ({
+          orderBy: mock(() => ({
+            limit: mock(() => Promise.resolve(newestRows)),
+          })),
+        })),
+      })),
+    })),
+    delete: mock(() => ({ where })),
+  };
+  return { db, deleteWhere: where };
+}
+
+describe("pruneStateTransitions", () => {
+  it("issues a delete that preserves the newest row when one exists", async () => {
+    const { db, deleteWhere } = makeMockDb([{ id: "newest-id" }]);
+    await pruneStateTransitions({
+      db: db as never,
+      systemId: "system-1",
+      rawRetentionDays: 7,
+    });
+    expect(db.select).toHaveBeenCalledTimes(1);
+    expect(db.delete).toHaveBeenCalledTimes(1);
+    expect(deleteWhere).toHaveBeenCalledTimes(1);
+  });
+
+  it("still issues a delete (cutoff-only) when the table is empty for the system", async () => {
+    const { db, deleteWhere } = makeMockDb([]);
+    await pruneStateTransitions({
+      db: db as never,
+      systemId: "system-1",
+      rawRetentionDays: 7,
+    });
+    expect(db.delete).toHaveBeenCalledTimes(1);
+    expect(deleteWhere).toHaveBeenCalledTimes(1);
+  });
+});

package/src/router.test.ts

@@ -72,6 +72,17 @@ describe("HealthCheck Router", () => {
     getSystem: mock(async () => null),
   };
 
+  const mockMaintenanceClient = {
+    hasActiveMaintenance: mock(async () => ({ active: false })),
+  };
+
+  const mockLogger = {
+    debug: mock(() => {}),
+    info: mock(() => {}),
+    warn: mock(() => {}),
+    error: mock(() => {}),
+  };
+
   const router = createHealthCheckRouter({
     database: mockDb as never,
     registry: mockRegistry,
@@ -81,6 +92,8 @@ describe("HealthCheck Router", () => {
     cache: passthroughCache,
     configService: mockConfigService as never,
     catalogClient: mockCatalogClient as never,
+    maintenanceClient: mockMaintenanceClient as never,
+    logger: mockLogger as never,
   });
 
   it("getStrategies returns strategies from registry", async () => {

package/src/router.ts

@@ -11,13 +11,20 @@ import {
 } from "@checkstack/backend-api";
 import { healthCheckContract } from "@checkstack/healthcheck-common";
 import type { StrategyCategory } from "@checkstack/healthcheck-common";
+import {
+  resolveResolutionRootFromStore,
+  resolveScriptPackagesDir,
+} from "@checkstack/script-packages-backend";
 import { HealthCheckService } from "./service";
+import { runCollectorScriptTest } from "./collector-script-test";
 import { healthCheckHooks } from "./hooks";
 import * as schema from "./schema";
 import { toJsonSchemaWithChartMeta } from "./schema-utils";
 import type { InferClient } from "@checkstack/common";
 import { GitOpsApi } from "@checkstack/gitops-common";
 import { CatalogApi } from "@checkstack/catalog-common";
+import { MaintenanceApi } from "@checkstack/maintenance-common";
+import type { Logger } from "@checkstack/backend-api";
 import type { HealthCheckCache } from "./cache";
 
 /**
@@ -35,6 +42,8 @@ export const createHealthCheckRouter = (opts: {
   cache: HealthCheckCache;
   configService: ConfigService;
   catalogClient: InferClient<typeof CatalogApi>;
+  maintenanceClient: InferClient<typeof MaintenanceApi>;
+  logger: Logger;
 }) => {
   const {
     database,
@@ -44,6 +53,8 @@ export const createHealthCheckRouter = (opts: {
     cache,
     configService,
     catalogClient,
+    maintenanceClient,
+    logger,
   } = opts;
   // Create service instance once - shared across all handlers
   const service = new HealthCheckService(
@@ -125,6 +136,19 @@ export const createHealthCheckRouter = (opts: {
       }));
     }),
 
+    testCollectorScript: os.testCollectorScript.handler(async ({ input }) => {
+      // Resolve the managed npm-package root from the local store so a
+      // collector test resolves the same allowlisted packages the real
+      // collector would (plan §4.1). Filesystem-only; safety is the
+      // runner's (auto-install disabled).
+      const status = await resolveResolutionRootFromStore(
+        resolveScriptPackagesDir(),
+      );
+      const resolutionRoot =
+        status.mode === "ready" ? status.root : undefined;
+      return runCollectorScriptTest({ input, deps: { resolutionRoot } });
+    }),
+
     getConfigurations: os.getConfigurations.handler(async () => {
       return { configurations: await service.getConfigurations() };
     }),
@@ -326,6 +350,26 @@ export const createHealthCheckRouter = (opts: {
       },
     ),
 
+    getHealthState: os.getHealthState.handler(async ({ input }) => {
+      return service.getHealthState({
+        systemId: input.systemId,
+        configurationId: input.configurationId,
+        transitionWindowMinutes: input.transitionWindowMinutes,
+        maintenanceClient,
+        logger,
+      });
+    }),
+
+    getBulkHealthState: os.getBulkHealthState.handler(async ({ input }) => {
+      const states = await service.getBulkHealthState({
+        systemIds: input.systemIds,
+        transitionWindowMinutes: input.transitionWindowMinutes,
+        maintenanceClient,
+        logger,
+      });
+      return { states };
+    }),
+
     // ========================================================================
     // SERVICE INTERFACE (S2S — satellite-backend)
     // ========================================================================

package/src/schema.ts

@@ -117,70 +117,50 @@ export const systemHealthChecks = pgTable(
 );
 
 /**
- * Records each time a check's *evaluated* state transitions from
- * non-unhealthy to unhealthy. Used to decide whether the per-check
- * incident threshold (N transitions in M minutes) has been met.
- * Pruned by the retention job alongside raw runs.
- */
-export const healthCheckUnhealthyTransitions = pgTable(
-  "health_check_unhealthy_transitions",
-  {
-    id: uuid("id").primaryKey().defaultRandom(),
-    configurationId: uuid("configuration_id")
-      .notNull()
-      .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
-    systemId: text("system_id").notNull(),
-    transitionedAt: timestamp("transitioned_at").defaultNow().notNull(),
-  },
-  (t) => ({
-    // Powers the threshold count query
-    // (WHERE config_id = ? AND system_id = ? AND transitioned_at > ?).
-    lookupIdx: index(
-      "health_check_unhealthy_transitions_lookup_idx",
-    ).on(t.configurationId, t.systemId, t.transitionedAt),
-  }),
-);
-
-/**
- * Mapping of auto-opened incidents back to the system + check that
- * triggered them. `closedAt` stays null while the incident is active;
- * the auto-close worker sets it once the linked system has been
- * steadily healthy for the cooldown.
+ * Records every *aggregate* health-status transition for a system
+ * (e.g. healthy -> degraded -> unhealthy -> healthy). This table is
+ * unconditional and covers ALL statuses, giving a reliable
+ * "in current status since" timestamp for arbitrary statuses.
+ *
+ * The former `health_check_unhealthy_transitions` table (per-check,
+ * unhealthy-only) was dropped: flapping is now counted in the automation
+ * engine's `automation_window_events` log via the windowed-count gate on the
+ * `system_health_changed` trigger, so healthcheck no longer keeps a separate
+ * transition audit for flapping.
  *
- * No FK to the incident table — that lives in another plugin's schema
- * and we treat it as a soft reference (incident deletes are handled
- * by the auto-close worker, which tolerates missing rows).
+ * One row is written per aggregate transition at the same point the
+ * `systemHealthChanged` hook fires. `configurationId` is the check that
+ * drove the transition (the just-ran check).
+ *
+ * Retention: pruned alongside raw runs, EXCEPT the single most-recent
+ * row per system is always kept so "in status since" never blanks for
+ * an active streak.
  */
-export const healthCheckAutoIncidents = pgTable(
-  "health_check_auto_incidents",
+export const healthCheckStateTransitions = pgTable(
+  "health_check_state_transitions",
   {
     id: uuid("id").primaryKey().defaultRandom(),
-    incidentId: uuid("incident_id").notNull(),
     systemId: text("system_id").notNull(),
+    /** The check whose run drove this aggregate transition. */
     configurationId: uuid("configuration_id")
       .notNull()
       .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
-    openedAt: timestamp("opened_at").defaultNow().notNull(),
-    closedAt: timestamp("closed_at"),
-    /**
-     * Auto-close cooldown snapshot taken when the incident was opened.
-     * `null` means "never auto-close" — the worker leaves this
-     * incident alone and an operator must resolve it manually. Stored
-     * per-row so a later policy change doesn't retroactively alter
-     * the close behaviour of incidents already in flight.
-     */
-    cooldownMinutes: integer("cooldown_minutes"),
+    fromStatus: healthCheckStatusEnum("from_status"),
+    toStatus: healthCheckStatusEnum("to_status").notNull(),
+    transitionedAt: timestamp("transitioned_at").defaultNow().notNull(),
   },
   (t) => ({
-    // Powers "is there an active auto-incident for this system?" check.
-    activeBySystemIdx: index(
-      "health_check_auto_incidents_active_by_system_idx",
-    ).on(t.systemId, t.closedAt),
-    // Powers "find the most recent close for this assignment" lookup
-    // used by the require-recovery-before-reopen check.
-    lastCloseByAssignmentIdx: index(
-      "health_check_auto_incidents_last_close_idx",
-    ).on(t.configurationId, t.systemId, t.closedAt),
+    // Powers "most recent transition into status X for this system"
+    // (WHERE system_id = ? AND to_status = ? ORDER BY transitioned_at DESC).
+    lookupIdx: index("health_check_state_transitions_lookup_idx").on(
+      t.systemId,
+      t.toStatus,
+      t.transitionedAt,
+    ),
+    // Powers the retention "keep newest per system" sweep.
+    systemRecentIdx: index(
+      "health_check_state_transitions_system_recent_idx",
+    ).on(t.systemId, t.transitionedAt),
   }),
 );
 

package/src/service-notification-policy.test.ts

@@ -51,9 +51,7 @@ describe("HealthCheckService.getAssignmentNotificationPolicy", () => {
 
   it("falls back to platform defaults when association exists but notificationPolicy is null", async () => {
     const customPlatformDefault: NotificationPolicy = {
-      ...DEFAULT_NOTIFICATION_POLICY,
-      autoCloseAfterMinutes: 120,
-      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 15 },
+      suppressDeEscalations: true,
     };
     const service = buildServiceWithRows(
       [{ notificationPolicy: null }],
@@ -63,40 +61,27 @@ describe("HealthCheckService.getAssignmentNotificationPolicy", () => {
       systemId: "sys-1",
       configurationId: "cfg-1",
     });
-    expect(policy.autoCloseAfterMinutes).toBe(120);
-    expect(policy.sustainedUnhealthyTrigger.durationMinutes).toBe(15);
+    expect(policy.suppressDeEscalations).toBe(true);
   });
 
   it("falls back to platform defaults when no association exists", async () => {
     const customPlatformDefault: NotificationPolicy = {
-      ...DEFAULT_NOTIFICATION_POLICY,
-      flappingTrigger: { enabled: true, transitions: 10, windowMinutes: 30 },
+      suppressDeEscalations: true,
     };
     const service = buildServiceWithRows([], customPlatformDefault);
     const policy = await service.getAssignmentNotificationPolicy({
       systemId: "sys-1",
       configurationId: "cfg-1",
     });
-    expect(policy.flappingTrigger).toEqual({
-      enabled: true,
-      transitions: 10,
-      windowMinutes: 30,
-    });
+    expect(policy).toEqual({ suppressDeEscalations: true });
   });
 
   it("prefers per-assignment override over platform defaults", async () => {
     const platformDefault: NotificationPolicy = {
-      ...DEFAULT_NOTIFICATION_POLICY,
-      autoOpenIncidentOnUnhealthy: false,
+      suppressDeEscalations: false,
     };
     const assignmentOverride = {
-      suppressDeEscalations: true,
-      autoOpenIncidentOnUnhealthy: true, // overrides platform default
-      useNotificationSuppression: true,
-      skipDuringMaintenance: true,
-      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 30 },
-      flappingTrigger: { enabled: true, transitions: 3, windowMinutes: 60 },
-      autoCloseAfterMinutes: 30,
+      suppressDeEscalations: true, // overrides platform default
     };
     const service = buildServiceWithRows(
       [{ notificationPolicy: assignmentOverride }],
@@ -106,69 +91,41 @@ describe("HealthCheckService.getAssignmentNotificationPolicy", () => {
       systemId: "sys-1",
       configurationId: "cfg-1",
     });
-    expect(policy.autoOpenIncidentOnUnhealthy).toBe(true);
     expect(policy.suppressDeEscalations).toBe(true);
   });
 
-  it("fills in defaults for partial stored policies", async () => {
-    // Older rows may have only `suppressDeEscalations` set from the
-    // first migration. All other fields must default in.
-    const service = buildServiceWithRows([
-      { notificationPolicy: { suppressDeEscalations: true } },
-    ]);
+  it("fills in defaults for an empty stored policy", async () => {
+    const service = buildServiceWithRows([{ notificationPolicy: {} }]);
     const policy = await service.getAssignmentNotificationPolicy({
       systemId: "sys-1",
       configurationId: "cfg-1",
     });
-    expect(policy.suppressDeEscalations).toBe(true);
-    expect(policy.autoOpenIncidentOnUnhealthy).toBe(true);
-    expect(policy.useNotificationSuppression).toBe(true);
-    expect(policy.skipDuringMaintenance).toBe(true);
-    expect(policy.sustainedUnhealthyTrigger).toEqual({
-      enabled: true,
-      durationMinutes: 30,
-    });
-    expect(policy.flappingTrigger).toEqual({
-      enabled: true,
-      transitions: 3,
-      windowMinutes: 60,
-    });
-    expect(policy.autoCloseAfterMinutes).toBe(30);
+    expect(policy).toEqual({ suppressDeEscalations: false });
   });
 
-  it("returns explicit values exactly when fully specified", async () => {
-    const service = buildServiceWithRows([
-      {
-        notificationPolicy: {
-          suppressDeEscalations: false,
-          autoOpenIncidentOnUnhealthy: false,
-          useNotificationSuppression: false,
-          skipDuringMaintenance: false,
-          sustainedUnhealthyTrigger: { enabled: false, durationMinutes: 15 },
-          flappingTrigger: {
-            enabled: true,
-            transitions: 5,
-            windowMinutes: 30,
-          },
-          autoCloseAfterMinutes: null,
-        },
+  it("strips removed legacy keys (auto-incident AND flapping) from stored rows without throwing", async () => {
+    // A row persisted before the legacy auto-incident fields and the flapping
+    // thresholds were removed still carries the larger object. The schema
+    // strips the dead keys and keeps the one surviving field.
+    const legacyOversizedRow = {
+      notificationPolicy: {
+        suppressDeEscalations: true,
+        // Removed flapping thresholds — moved onto the automation trigger.
+        flappingTrigger: { enabled: true, transitions: 7, windowMinutes: 45 },
+        // Removed legacy auto-incident keys — must be dropped, not rejected.
+        autoOpenIncidentOnUnhealthy: true,
+        useNotificationSuppression: true,
+        skipDuringMaintenance: true,
+        sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 15 },
+        autoCloseAfterMinutes: 120,
       },
-    ]);
+    };
+    const service = buildServiceWithRows([legacyOversizedRow]);
     const policy = await service.getAssignmentNotificationPolicy({
       systemId: "sys-1",
       configurationId: "cfg-1",
     });
-    expect(policy.autoOpenIncidentOnUnhealthy).toBe(false);
-    expect(policy.skipDuringMaintenance).toBe(false);
-    expect(policy.sustainedUnhealthyTrigger).toEqual({
-      enabled: false,
-      durationMinutes: 15,
-    });
-    expect(policy.flappingTrigger).toEqual({
-      enabled: true,
-      transitions: 5,
-      windowMinutes: 30,
-    });
-    expect(policy.autoCloseAfterMinutes).toBeNull();
+    expect(policy).toEqual({ suppressDeEscalations: true });
+    expect(Object.keys(policy)).toEqual(["suppressDeEscalations"]);
   });
 });

package/src/service.ts

@@ -38,7 +38,10 @@ import {
 } from "drizzle-orm";
 import { ORPCError } from "@orpc/server";
 import { evaluateHealthStatus } from "./state-evaluator";
+import { computeHealthState, type HealthState } from "./health-state";
 import { stateThresholds } from "./state-thresholds-migrations";
+import type { MaintenanceApi } from "@checkstack/maintenance-common";
+import type { Logger } from "@checkstack/backend-api";
 import { incrementHourlyAggregate } from "./realtime-aggregation";
 import type {
   HealthCheckRegistry,
@@ -62,6 +65,10 @@ type Db = SafeDatabase<typeof schema>;
 // satellite assignment run-context. Optional on the service.
 type CatalogClient = InferClient<typeof CatalogApi>;
 
+// Maintenance client type used to fold suppression-agnostic maintenance
+// state into the health-state snapshot. Optional on the read path.
+type MaintenanceClient = InferClient<typeof MaintenanceApi>;
+
 interface SystemCheckStatus {
   configurationId: string;
   configurationName: string;
@@ -558,6 +565,88 @@ export class HealthCheckService {
     };
   }
 
+  /**
+   * Live health-state snapshot for a single system (Wave-2 sensing
+   * contract). When `configurationId` is given, status reflects that
+   * one check; otherwise it is the aggregate. `inStatusSince` /
+   * `inStatusForMs` come from the state-transitions table, latency from
+   * the newest run, windowed metrics from hourly aggregates, and
+   * `inMaintenance` from the maintenance plugin (suppression-agnostic,
+   * fail-open). `now` is threaded so bulk reads share one timestamp.
+   */
+  async getHealthState({
+    systemId,
+    configurationId,
+    maintenanceClient,
+    logger,
+    transitionWindowMinutes,
+    now,
+  }: {
+    systemId: string;
+    configurationId?: string;
+    maintenanceClient?: MaintenanceClient;
+    logger?: Logger;
+    transitionWindowMinutes?: number;
+    now?: Date;
+  }): Promise<HealthState> {
+    return computeHealthState({
+      db: this.db,
+      systemId,
+      configurationId,
+      maintenanceClient,
+      logger,
+      transitionWindowMinutes,
+      now,
+      resolveStatus: async () => {
+        const overview = await this.getSystemHealthStatus(systemId);
+        if (!configurationId) return overview.status;
+        const check = overview.checkStatuses.find(
+          (c) => c.configurationId === configurationId,
+        );
+        // Unknown check id -> treat as healthy (no signal), mirroring
+        // the "no checks configured" default.
+        return check?.status ?? "healthy";
+      },
+    });
+  }
+
+  /**
+   * Bulk variant of {@link getHealthState}. Resolves every system in
+   * parallel against a single shared `now` so durations are consistent
+   * across the batch. Avoids N+1 from dashboards and multi-system
+   * automation rules.
+   */
+  async getBulkHealthState({
+    systemIds,
+    maintenanceClient,
+    logger,
+    transitionWindowMinutes,
+    now = new Date(),
+  }: {
+    systemIds: string[];
+    maintenanceClient?: MaintenanceClient;
+    logger?: Logger;
+    transitionWindowMinutes?: number;
+    now?: Date;
+  }): Promise<Record<string, HealthState>> {
+    const entries = await Promise.all(
+      systemIds.map(
+        async (systemId) =>
+          [
+            systemId,
+            await this.getHealthState({
+              systemId,
+              maintenanceClient,
+              logger,
+              transitionWindowMinutes,
+              now,
+            }),
+          ] as const,
+      ),
+    );
+    return Object.fromEntries(entries);
+  }
+
   /**
    * Get comprehensive health overview for a system.
    * Returns all health checks with their last 25 runs for sparkline visualization.