@checkstack/healthcheck-backend 1.2.0 → 1.4.0

package/src/retention-job.ts

@@ -4,9 +4,10 @@ import {
   healthCheckRuns,
   systemHealthChecks,
   healthCheckAggregates,
+  healthCheckStateTransitions,
   DEFAULT_RETENTION_CONFIG,
 } from "./schema";
-import { eq, and, lt, sql } from "drizzle-orm";
+import { eq, and, lt, sql, desc } from "drizzle-orm";
 import type { QueueManager } from "@checkstack/queue-api";
 
 type Db = SafeDatabase<typeof schema>;
@@ -69,6 +70,29 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
   // Get all unique system-config assignments
   const assignments = await db.select().from(systemHealthChecks);
 
+  // State transitions are system-level (not per-config), so prune them
+  // once per unique system rather than once per assignment. Use the
+  // longest rawRetentionDays among the system's assignments so a single
+  // short-retention check can't drop history another check still wants.
+  const rawRetentionBySystem = new Map<string, number>();
+  for (const assignment of assignments) {
+    const days = (
+      assignment.retentionConfig ?? DEFAULT_RETENTION_CONFIG
+    ).rawRetentionDays;
+    const existing = rawRetentionBySystem.get(assignment.systemId);
+    if (existing === undefined || days > existing) {
+      rawRetentionBySystem.set(assignment.systemId, days);
+    }
+  }
+
+  for (const [systemId, rawRetentionDays] of rawRetentionBySystem) {
+    try {
+      await pruneStateTransitions({ db, systemId, rawRetentionDays });
+    } catch (error) {
+      logger.error(`State-transition prune failed for ${systemId}`, { error });
+    }
+  }
+
   for (const assignment of assignments) {
     const retentionConfig =
       assignment.retentionConfig ?? DEFAULT_RETENTION_CONFIG;
@@ -134,6 +158,46 @@ async function deleteExpiredRawRuns(params: DeleteExpiredRawRunsParams) {
     );
 }
 
+interface PruneStateTransitionsParams {
+  db: Db;
+  systemId: string;
+  rawRetentionDays: number;
+}
+
+/**
+ * Prune aggregate state-transition rows older than the raw-run retention
+ * window, but ALWAYS keep the single most-recent transition for the
+ * system so "in current status since" never blanks for an active streak
+ * (decision D3). Deletes rows that are both older than the cutoff AND
+ * not the newest row.
+ */
+export async function pruneStateTransitions(
+  params: PruneStateTransitionsParams,
+): Promise<void> {
+  const { db, systemId, rawRetentionDays } = params;
+
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
+
+  const [newest] = await db
+    .select({ id: healthCheckStateTransitions.id })
+    .from(healthCheckStateTransitions)
+    .where(eq(healthCheckStateTransitions.systemId, systemId))
+    .orderBy(desc(healthCheckStateTransitions.transitionedAt))
+    .limit(1);
+
+  const conditions = [
+    eq(healthCheckStateTransitions.systemId, systemId),
+    lt(healthCheckStateTransitions.transitionedAt, cutoffDate),
+  ];
+  if (newest) {
+    // Never delete the most-recent row, even if it predates the cutoff.
+    conditions.push(sql`${healthCheckStateTransitions.id} <> ${newest.id}`);
+  }
+
+  await db.delete(healthCheckStateTransitions).where(and(...conditions));
+}
+
 interface RollupParams {
   db: Db;
   systemId: string;

package/src/retention-state-transitions.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, mock } from "bun:test";
+import { pruneStateTransitions } from "./retention-job";
+
+/**
+ * Mock db exposing the two shapes pruneStateTransitions uses:
+ * - select(...).from(...).where(...).orderBy(...).limit(...) -> newest row
+ * - delete(...).where(...) -> prune
+ */
+function makeMockDb(newestRows: Array<{ id: string }>) {
+  const where = mock(() => Promise.resolve());
+  const db = {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() => ({
+          orderBy: mock(() => ({
+            limit: mock(() => Promise.resolve(newestRows)),
+          })),
+        })),
+      })),
+    })),
+    delete: mock(() => ({ where })),
+  };
+  return { db, deleteWhere: where };
+}
+
+describe("pruneStateTransitions", () => {
+  it("issues a delete that preserves the newest row when one exists", async () => {
+    const { db, deleteWhere } = makeMockDb([{ id: "newest-id" }]);
+    await pruneStateTransitions({
+      db: db as never,
+      systemId: "system-1",
+      rawRetentionDays: 7,
+    });
+    expect(db.select).toHaveBeenCalledTimes(1);
+    expect(db.delete).toHaveBeenCalledTimes(1);
+    expect(deleteWhere).toHaveBeenCalledTimes(1);
+  });
+
+  it("still issues a delete (cutoff-only) when the table is empty for the system", async () => {
+    const { db, deleteWhere } = makeMockDb([]);
+    await pruneStateTransitions({
+      db: db as never,
+      systemId: "system-1",
+      rawRetentionDays: 7,
+    });
+    expect(db.delete).toHaveBeenCalledTimes(1);
+    expect(deleteWhere).toHaveBeenCalledTimes(1);
+  });
+});

package/src/router.test.ts

@@ -68,6 +68,21 @@ describe("HealthCheck Router", () => {
     getRedacted: mock(async () => undefined),
   };
 
+  const mockCatalogClient = {
+    getSystem: mock(async () => null),
+  };
+
+  const mockMaintenanceClient = {
+    hasActiveMaintenance: mock(async () => ({ active: false })),
+  };
+
+  const mockLogger = {
+    debug: mock(() => {}),
+    info: mock(() => {}),
+    warn: mock(() => {}),
+    error: mock(() => {}),
+  };
+
   const router = createHealthCheckRouter({
     database: mockDb as never,
     registry: mockRegistry,
@@ -76,6 +91,9 @@ describe("HealthCheck Router", () => {
     getEmitHook: () => undefined,
     cache: passthroughCache,
     configService: mockConfigService as never,
+    catalogClient: mockCatalogClient as never,
+    maintenanceClient: mockMaintenanceClient as never,
+    logger: mockLogger as never,
   });
 
   it("getStrategies returns strategies from registry", async () => {

package/src/router.ts

@@ -11,12 +11,20 @@ import {
 } from "@checkstack/backend-api";
 import { healthCheckContract } from "@checkstack/healthcheck-common";
 import type { StrategyCategory } from "@checkstack/healthcheck-common";
+import {
+  resolveResolutionRootFromStore,
+  resolveScriptPackagesDir,
+} from "@checkstack/script-packages-backend";
 import { HealthCheckService } from "./service";
+import { runCollectorScriptTest } from "./collector-script-test";
 import { healthCheckHooks } from "./hooks";
 import * as schema from "./schema";
 import { toJsonSchemaWithChartMeta } from "./schema-utils";
 import type { InferClient } from "@checkstack/common";
 import { GitOpsApi } from "@checkstack/gitops-common";
+import { CatalogApi } from "@checkstack/catalog-common";
+import { MaintenanceApi } from "@checkstack/maintenance-common";
+import type { Logger } from "@checkstack/backend-api";
 import type { HealthCheckCache } from "./cache";
 
 /**
@@ -33,14 +41,28 @@ export const createHealthCheckRouter = (opts: {
   getEmitHook: () => ((hook: { id: string }, payload: Record<string, unknown>) => Promise<void>) | undefined;
   cache: HealthCheckCache;
   configService: ConfigService;
+  catalogClient: InferClient<typeof CatalogApi>;
+  maintenanceClient: InferClient<typeof MaintenanceApi>;
+  logger: Logger;
 }) => {
-  const { database, registry, collectorRegistry, getEmitHook, cache, configService } = opts;
+  const {
+    database,
+    registry,
+    collectorRegistry,
+    getEmitHook,
+    cache,
+    configService,
+    catalogClient,
+    maintenanceClient,
+    logger,
+  } = opts;
   // Create service instance once - shared across all handlers
   const service = new HealthCheckService(
     database,
     registry,
     collectorRegistry,
     configService,
+    catalogClient,
   );
 
   // Create contract implementer with context type AND auto auth middleware
@@ -114,6 +136,19 @@ export const createHealthCheckRouter = (opts: {
       }));
     }),
 
+    testCollectorScript: os.testCollectorScript.handler(async ({ input }) => {
+      // Resolve the managed npm-package root from the local store so a
+      // collector test resolves the same allowlisted packages the real
+      // collector would (plan §4.1). Filesystem-only; safety is the
+      // runner's (auto-install disabled).
+      const status = await resolveResolutionRootFromStore(
+        resolveScriptPackagesDir(),
+      );
+      const resolutionRoot =
+        status.mode === "ready" ? status.root : undefined;
+      return runCollectorScriptTest({ input, deps: { resolutionRoot } });
+    }),
+
     getConfigurations: os.getConfigurations.handler(async () => {
       return { configurations: await service.getConfigurations() };
     }),
@@ -315,6 +350,26 @@ export const createHealthCheckRouter = (opts: {
       },
     ),
 
+    getHealthState: os.getHealthState.handler(async ({ input }) => {
+      return service.getHealthState({
+        systemId: input.systemId,
+        configurationId: input.configurationId,
+        transitionWindowMinutes: input.transitionWindowMinutes,
+        maintenanceClient,
+        logger,
+      });
+    }),
+
+    getBulkHealthState: os.getBulkHealthState.handler(async ({ input }) => {
+      const states = await service.getBulkHealthState({
+        systemIds: input.systemIds,
+        transitionWindowMinutes: input.transitionWindowMinutes,
+        maintenanceClient,
+        logger,
+      });
+      return { states };
+    }),
+
     // ========================================================================
     // SERVICE INTERFACE (S2S — satellite-backend)
     // ========================================================================

package/src/schema.ts

@@ -117,70 +117,50 @@ export const systemHealthChecks = pgTable(
 );
 
 /**
- * Records each time a check's *evaluated* state transitions from
- * non-unhealthy to unhealthy. Used to decide whether the per-check
- * incident threshold (N transitions in M minutes) has been met.
- * Pruned by the retention job alongside raw runs.
- */
-export const healthCheckUnhealthyTransitions = pgTable(
-  "health_check_unhealthy_transitions",
-  {
-    id: uuid("id").primaryKey().defaultRandom(),
-    configurationId: uuid("configuration_id")
-      .notNull()
-      .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
-    systemId: text("system_id").notNull(),
-    transitionedAt: timestamp("transitioned_at").defaultNow().notNull(),
-  },
-  (t) => ({
-    // Powers the threshold count query
-    // (WHERE config_id = ? AND system_id = ? AND transitioned_at > ?).
-    lookupIdx: index(
-      "health_check_unhealthy_transitions_lookup_idx",
-    ).on(t.configurationId, t.systemId, t.transitionedAt),
-  }),
-);
-
-/**
- * Mapping of auto-opened incidents back to the system + check that
- * triggered them. `closedAt` stays null while the incident is active;
- * the auto-close worker sets it once the linked system has been
- * steadily healthy for the cooldown.
+ * Records every *aggregate* health-status transition for a system
+ * (e.g. healthy -> degraded -> unhealthy -> healthy). This table is
+ * unconditional and covers ALL statuses, giving a reliable
+ * "in current status since" timestamp for arbitrary statuses.
+ *
+ * The former `health_check_unhealthy_transitions` table (per-check,
+ * unhealthy-only) was dropped: flapping is now counted in the automation
+ * engine's `automation_window_events` log via the windowed-count gate on the
+ * `system_health_changed` trigger, so healthcheck no longer keeps a separate
+ * transition audit for flapping.
  *
- * No FK to the incident table — that lives in another plugin's schema
- * and we treat it as a soft reference (incident deletes are handled
- * by the auto-close worker, which tolerates missing rows).
+ * One row is written per aggregate transition at the same point the
+ * `systemHealthChanged` hook fires. `configurationId` is the check that
+ * drove the transition (the just-ran check).
+ *
+ * Retention: pruned alongside raw runs, EXCEPT the single most-recent
+ * row per system is always kept so "in status since" never blanks for
+ * an active streak.
  */
-export const healthCheckAutoIncidents = pgTable(
-  "health_check_auto_incidents",
+export const healthCheckStateTransitions = pgTable(
+  "health_check_state_transitions",
   {
     id: uuid("id").primaryKey().defaultRandom(),
-    incidentId: uuid("incident_id").notNull(),
     systemId: text("system_id").notNull(),
+    /** The check whose run drove this aggregate transition. */
     configurationId: uuid("configuration_id")
       .notNull()
       .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
-    openedAt: timestamp("opened_at").defaultNow().notNull(),
-    closedAt: timestamp("closed_at"),
-    /**
-     * Auto-close cooldown snapshot taken when the incident was opened.
-     * `null` means "never auto-close" — the worker leaves this
-     * incident alone and an operator must resolve it manually. Stored
-     * per-row so a later policy change doesn't retroactively alter
-     * the close behaviour of incidents already in flight.
-     */
-    cooldownMinutes: integer("cooldown_minutes"),
+    fromStatus: healthCheckStatusEnum("from_status"),
+    toStatus: healthCheckStatusEnum("to_status").notNull(),
+    transitionedAt: timestamp("transitioned_at").defaultNow().notNull(),
   },
   (t) => ({
-    // Powers "is there an active auto-incident for this system?" check.
-    activeBySystemIdx: index(
-      "health_check_auto_incidents_active_by_system_idx",
-    ).on(t.systemId, t.closedAt),
-    // Powers "find the most recent close for this assignment" lookup
-    // used by the require-recovery-before-reopen check.
-    lastCloseByAssignmentIdx: index(
-      "health_check_auto_incidents_last_close_idx",
-    ).on(t.configurationId, t.systemId, t.closedAt),
+    // Powers "most recent transition into status X for this system"
+    // (WHERE system_id = ? AND to_status = ? ORDER BY transitioned_at DESC).
+    lookupIdx: index("health_check_state_transitions_lookup_idx").on(
+      t.systemId,
+      t.toStatus,
+      t.transitionedAt,
+    ),
+    // Powers the retention "keep newest per system" sweep.
+    systemRecentIdx: index(
+      "health_check_state_transitions_system_recent_idx",
+    ).on(t.systemId, t.transitionedAt),
   }),
 );
 

package/src/service-assignments.test.ts

@@ -0,0 +1,184 @@
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+import { HealthCheckService } from "./service";
+
+/**
+ * Tests for getAssignmentsForSatellite run-context population:
+ * - assignments carry configName (from the config row's name)
+ * - systemName resolves via the optional catalog client, falling back to
+ *   systemId when no client is wired or the lookup fails.
+ */
+describe("HealthCheckService.getAssignmentsForSatellite", () => {
+  const SATELLITE_ID = "sat-1";
+
+  type Association = {
+    systemId: string;
+    configurationId: string;
+    satelliteIds: string[] | null;
+    enabled: boolean;
+  };
+
+  type Config = {
+    id: string;
+    name: string;
+    strategyId: string;
+    config: Record<string, unknown>;
+    collectors: unknown[] | null;
+    intervalSeconds: number;
+    paused: boolean;
+  };
+
+  let associations: Association[] = [];
+  let configs: Config[] = [];
+
+  /**
+   * Mock db: the method issues two distinct select shapes:
+   * - associations: .select({...}).from(systemHealthChecks)  -> awaited array
+   * - config:        .select().from(...).where(...)           -> awaited array
+   * We disambiguate by call order: the first select() resolves associations,
+   * subsequent select().from().where() resolve a single matching config.
+   */
+  function createMockDb() {
+    let firstSelect = true;
+    return {
+      select: mock(() => {
+        if (firstSelect) {
+          firstSelect = false;
+          return {
+            from: mock(() => Promise.resolve([...associations])),
+          };
+        }
+        return {
+          from: mock(() => ({
+            where: mock(() => {
+              // Return the next unmatched config in order; the loop fetches
+              // one config per matching association.
+              return Promise.resolve(configs.length > 0 ? [configs[0]] : []);
+            }),
+          })),
+        };
+      }),
+    };
+  }
+
+  beforeEach(() => {
+    associations = [];
+    configs = [];
+  });
+
+  it("populates configName and resolves systemName via the catalog client", async () => {
+    associations = [
+      {
+        systemId: "system-1",
+        configurationId: "config-1",
+        satelliteIds: [SATELLITE_ID],
+        enabled: true,
+      },
+    ];
+    configs = [
+      {
+        id: "config-1",
+        name: "API health",
+        strategyId: "http",
+        config: { url: "https://example.com" },
+        collectors: null,
+        intervalSeconds: 60,
+        paused: false,
+      },
+    ];
+
+    const getSystem = mock(() =>
+      Promise.resolve({ id: "system-1", name: "Production API" }),
+    );
+    const catalogClient = { getSystem } as never;
+
+    const mockDb = createMockDb();
+    const service = new HealthCheckService(
+      mockDb as never,
+      {} as never,
+      {} as never,
+      undefined,
+      catalogClient,
+    );
+
+    const result = await service.getAssignmentsForSatellite(SATELLITE_ID);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].configName).toBe("API health");
+    expect(result[0].systemName).toBe("Production API");
+    expect(getSystem).toHaveBeenCalledWith({ systemId: "system-1" });
+  });
+
+  it("falls back to systemId when no catalog client is provided", async () => {
+    associations = [
+      {
+        systemId: "system-1",
+        configurationId: "config-1",
+        satelliteIds: [SATELLITE_ID],
+        enabled: true,
+      },
+    ];
+    configs = [
+      {
+        id: "config-1",
+        name: "API health",
+        strategyId: "http",
+        config: {},
+        collectors: null,
+        intervalSeconds: 30,
+        paused: false,
+      },
+    ];
+
+    const mockDb = createMockDb();
+    const service = new HealthCheckService(
+      mockDb as never,
+      {} as never,
+      {} as never,
+    );
+
+    const result = await service.getAssignmentsForSatellite(SATELLITE_ID);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].configName).toBe("API health");
+    expect(result[0].systemName).toBe("system-1");
+  });
+
+  it("falls back to systemId when the catalog lookup throws", async () => {
+    associations = [
+      {
+        systemId: "system-1",
+        configurationId: "config-1",
+        satelliteIds: [SATELLITE_ID],
+        enabled: true,
+      },
+    ];
+    configs = [
+      {
+        id: "config-1",
+        name: "API health",
+        strategyId: "http",
+        config: {},
+        collectors: null,
+        intervalSeconds: 60,
+        paused: false,
+      },
+    ];
+
+    const getSystem = mock(() => Promise.reject(new Error("catalog down")));
+    const catalogClient = { getSystem } as never;
+
+    const mockDb = createMockDb();
+    const service = new HealthCheckService(
+      mockDb as never,
+      {} as never,
+      {} as never,
+      undefined,
+      catalogClient,
+    );
+
+    const result = await service.getAssignmentsForSatellite(SATELLITE_ID);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].systemName).toBe("system-1");
+  });
+});