@checkstack/healthcheck-backend 0.4.2 → 0.6.0

package/src/queue-executor.ts

@@ -18,9 +18,11 @@ import { type SignalService } from "@checkstack/signal-common";
 import {
   HEALTH_CHECK_RUN_COMPLETED,
   type HealthCheckStatus,
+  stripEphemeralFields,
 } from "@checkstack/healthcheck-common";
 import { CatalogApi, catalogRoutes } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
+import { IncidentApi } from "@checkstack/incident-common";
 import { resolveRoute, type InferClient } from "@checkstack/common";
 import { HealthCheckService } from "./service";
 import { healthCheckHooks } from "./hooks";
@@ -28,6 +30,7 @@ import { healthCheckHooks } from "./hooks";
 type Db = SafeDatabase<typeof schema>;
 type CatalogClient = InferClient<typeof CatalogApi>;
 type MaintenanceClient = InferClient<typeof MaintenanceApi>;
+type IncidentClient = InferClient<typeof IncidentApi>;
 
 /**
  * Payload for health check queue jobs
@@ -89,7 +92,7 @@ export async function scheduleHealthCheck(props: {
 
 /**
  * Notify system subscribers about a health state change.
- * Skips notification if the system has active maintenance with suppression enabled.
+ * Skips notification if the system has active maintenance or incident with suppression enabled.
  */
 async function notifyStateChange(props: {
   systemId: string;
@@ -97,6 +100,7 @@ async function notifyStateChange(props: {
   newStatus: HealthCheckStatus;
   catalogClient: CatalogClient;
   maintenanceClient: MaintenanceClient;
+  incidentClient: IncidentClient;
   logger: Logger;
 }): Promise<void> {
   const {
@@ -105,6 +109,7 @@ async function notifyStateChange(props: {
     newStatus,
     catalogClient,
     maintenanceClient,
+    incidentClient,
     logger,
   } = props;
 
@@ -131,6 +136,24 @@ async function notifyStateChange(props: {
     );
   }
 
+  // Check if notifications should be suppressed due to active incident
+  try {
+    const { suppressed } =
+      await incidentClient.hasActiveIncidentWithSuppression({ systemId });
+    if (suppressed) {
+      logger.debug(
+        `Skipping notification for ${systemId}: active incident with suppression enabled`,
+      );
+      return;
+    }
+  } catch (error) {
+    // Log but continue with notification - suppression check failure shouldn't block notifications
+    logger.warn(
+      `Failed to check incident suppression for ${systemId}, proceeding with notification:`,
+      error,
+    );
+  }
+
   const isRecovery = newStatus === "healthy" && previousStatus !== "healthy";
   const isDegraded = newStatus === "degraded";
   const isUnhealthy = newStatus === "unhealthy";
@@ -195,6 +218,7 @@ async function executeHealthCheckJob(props: {
   signalService: SignalService;
   catalogClient: CatalogClient;
   maintenanceClient: MaintenanceClient;
+  incidentClient: IncidentClient;
   getEmitHook: () => EmitHookFn | undefined;
 }): Promise<void> {
   const {
@@ -206,6 +230,7 @@ async function executeHealthCheckJob(props: {
     signalService,
     catalogClient,
     maintenanceClient,
+    incidentClient,
     getEmitHook,
   } = props;
   const { configId, systemId } = payload;
@@ -228,6 +253,7 @@ async function executeHealthCheckJob(props: {
         collectors: healthCheckConfigurations.collectors,
         interval: healthCheckConfigurations.intervalSeconds,
         enabled: systemHealthChecks.enabled,
+        paused: healthCheckConfigurations.paused,
       })
       .from(systemHealthChecks)
       .innerJoin(
@@ -250,6 +276,14 @@ async function executeHealthCheckJob(props: {
       return;
     }
 
+    // If configuration is paused, skip execution (job continues to be scheduled)
+    if (configRow.paused) {
+      logger.debug(
+        `Health check ${configId} is paused, skipping execution for system ${systemId}`,
+      );
+      return;
+    }
+
     // Fetch system name for signal payload
     let systemName = systemId;
     try {
@@ -321,6 +355,7 @@ async function executeHealthCheckJob(props: {
           newStatus: newState.status,
           catalogClient,
           maintenanceClient,
+          incidentClient,
           logger,
         });
       }
@@ -388,11 +423,17 @@ async function executeHealthCheckJob(props: {
             }
           }
 
+          // Strip ephemeral fields (like HTTP body) before storage to save space
+          const strippedResult = stripEphemeralFields(
+            collectorResult.result as Record<string, unknown>,
+            registered.collector.result.schema,
+          );
+
           // Store result under the collector's UUID, with collector type and assertion metadata
           collectorResults[storageKey] = {
             _collectorId: collectorEntry.collectorId, // Store the type for frontend schema linking
             _assertionFailed: assertionFailed, // null if no assertion failed
-            ...collectorResult.result,
+            ...strippedResult,
           };
         } catch (error) {
           hasCollectorError = true;
@@ -463,6 +504,7 @@ async function executeHealthCheckJob(props: {
         newStatus: newState.status,
         catalogClient,
         maintenanceClient,
+        incidentClient,
         logger,
       });
 
@@ -557,6 +599,7 @@ async function executeHealthCheckJob(props: {
         newStatus: newState.status,
         catalogClient,
         maintenanceClient,
+        incidentClient,
         logger,
       });
 
@@ -612,6 +655,7 @@ export async function setupHealthCheckWorker(props: {
   signalService: SignalService;
   catalogClient: CatalogClient;
   maintenanceClient: MaintenanceClient;
+  incidentClient: IncidentClient;
   getEmitHook: () => EmitHookFn | undefined;
 }): Promise<void> {
   const {
@@ -623,6 +667,7 @@ export async function setupHealthCheckWorker(props: {
     signalService,
     catalogClient,
     maintenanceClient,
+    incidentClient,
     getEmitHook,
   } = props;
 
@@ -641,6 +686,7 @@ export async function setupHealthCheckWorker(props: {
         signalService,
         catalogClient,
         maintenanceClient,
+        incidentClient,
         getEmitHook,
       });
     },

package/src/retention-job.ts

@@ -2,6 +2,7 @@ import type {
   HealthCheckRegistry,
   Logger,
   SafeDatabase,
+  CollectorRegistry,
 } from "@checkstack/backend-api";
 import * as schema from "./schema";
 import {
@@ -13,12 +14,19 @@ import {
 } from "./schema";
 import { eq, and, lt, sql } from "drizzle-orm";
 import type { QueueManager } from "@checkstack/queue-api";
+import {
+  aggregateCollectorData,
+  calculateLatencyStats,
+  countStatuses,
+  extractLatencies,
+} from "./aggregation-utils";
 
 type Db = SafeDatabase<typeof schema>;
 
 interface RetentionJobDeps {
   db: Db;
   registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
   logger: Logger;
   queueManager: QueueManager;
 }
@@ -36,7 +44,7 @@ interface RetentionJobPayload {
  * 3. Deletes expired daily aggregates
  */
 export async function setupRetentionJob(deps: RetentionJobDeps) {
-  const { queueManager, logger, db, registry } = deps;
+  const { queueManager, logger, db, registry, collectorRegistry } = deps;
 
   const queue = queueManager.getQueue<RetentionJobPayload>(RETENTION_QUEUE);
 
@@ -44,7 +52,13 @@ export async function setupRetentionJob(deps: RetentionJobDeps) {
   await queue.consume(
     async () => {
       logger.info("Starting health check retention job");
-      await runRetentionJob({ db, registry, logger, queueManager });
+      await runRetentionJob({
+        db,
+        registry,
+        collectorRegistry,
+        logger,
+        queueManager,
+      });
       logger.info("Completed health check retention job");
     },
     { consumerGroup: "retention-worker" },
@@ -66,7 +80,7 @@ export async function setupRetentionJob(deps: RetentionJobDeps) {
  * Main retention job logic
  */
 export async function runRetentionJob(deps: RetentionJobDeps) {
-  const { db, registry, logger } = deps;
+  const { db, registry, collectorRegistry, logger } = deps;
 
   // Get all unique system-config assignments
   const assignments = await db.select().from(systemHealthChecks);
@@ -80,6 +94,7 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
       await aggregateRawRuns({
         db,
         registry,
+        collectorRegistry,
         systemId: assignment.systemId,
         configurationId: assignment.configurationId,
         rawRetentionDays: retentionConfig.rawRetentionDays,
@@ -112,6 +127,7 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
 interface AggregateRawRunsParams {
   db: Db;
   registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
   systemId: string;
   configurationId: string;
   rawRetentionDays: number;
@@ -121,7 +137,14 @@ interface AggregateRawRunsParams {
  * Aggregates raw runs older than retention period into hourly buckets
  */
 async function aggregateRawRuns(params: AggregateRawRunsParams) {
-  const { db, registry, systemId, configurationId, rawRetentionDays } = params;
+  const {
+    db,
+    registry,
+    collectorRegistry,
+    systemId,
+    configurationId,
+    rawRetentionDays,
+  } = params;
 
   const cutoffDate = new Date();
   cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
@@ -183,42 +206,43 @@ async function aggregateRawRuns(params: AggregateRawRunsParams) {
 
   // Create aggregates and delete raw runs
   for (const [, bucket] of buckets) {
-    // Calculate metrics
     const runCount = bucket.runs.length;
-    let healthyCount = 0;
-    let degradedCount = 0;
-    let unhealthyCount = 0;
-    for (const r of bucket.runs) {
-      if (r.status === "healthy") healthyCount++;
-      if (r.status === "degraded") degradedCount++;
-      if (r.status === "unhealthy") unhealthyCount++;
-    }
 
-    const latencies = bucket.runs
-      .map((r) => r.latencyMs)
-      .filter((l): l is number => l !== undefined);
-
-    let avgLatencyMs: number | undefined;
-    let minLatencyMs: number | undefined;
-    let maxLatencyMs: number | undefined;
-    let p95LatencyMs: number | undefined;
-
-    if (latencies.length > 0) {
-      let sum = 0;
-      for (const l of latencies) sum += l;
-      avgLatencyMs = Math.round(sum / latencies.length);
-      minLatencyMs = Math.min(...latencies);
-      maxLatencyMs = Math.max(...latencies);
-      p95LatencyMs = calculatePercentile(latencies, 95);
-    }
+    // Calculate status counts
+    const { healthyCount, degradedCount, unhealthyCount } = countStatuses(
+      bucket.runs,
+    );
 
-    // Aggregate result if strategy is available
+    // Calculate latency stats
+    const latencies = extractLatencies(bucket.runs);
+    const {
+      latencySumMs,
+      avgLatencyMs,
+      minLatencyMs,
+      maxLatencyMs,
+      p95LatencyMs,
+    } = calculateLatencyStats(latencies);
+
+    // Aggregate strategy result
     let aggregatedResult: Record<string, unknown> | undefined;
     if (strategy) {
-      aggregatedResult = strategy.aggregateResult(bucket.runs) as Record<
+      const strategyResult = strategy.aggregateResult(bucket.runs) as Record<
         string,
         unknown
       >;
+
+      // Aggregate collector data
+      const collectorsAggregated = aggregateCollectorData(
+        bucket.runs,
+        collectorRegistry,
+      );
+
+      aggregatedResult = {
+        ...strategyResult,
+        ...(Object.keys(collectorsAggregated).length > 0
+          ? { collectors: collectorsAggregated }
+          : {}),
+      };
     }
 
     // Insert or update aggregate
@@ -233,6 +257,7 @@ async function aggregateRawRuns(params: AggregateRawRunsParams) {
         healthyCount,
         degradedCount,
         unhealthyCount,
+        latencySumMs,
         avgLatencyMs,
         minLatencyMs,
         maxLatencyMs,
@@ -319,20 +344,23 @@ async function rollupHourlyAggregates(params: RollupParams) {
     let healthyCount = 0;
     let degradedCount = 0;
     let unhealthyCount = 0;
-    let totalWeightedLatency = 0;
+    let latencySumMs = 0;
 
     for (const a of bucket.aggregates) {
       runCount += a.runCount;
       healthyCount += a.healthyCount;
       degradedCount += a.degradedCount;
       unhealthyCount += a.unhealthyCount;
-      if (a.avgLatencyMs !== null) {
-        totalWeightedLatency += a.avgLatencyMs * a.runCount;
+      // Use latencySumMs if available, fallback to avg*count approximation
+      if (a.latencySumMs !== null) {
+        latencySumMs += a.latencySumMs;
+      } else if (a.avgLatencyMs !== null) {
+        latencySumMs += a.avgLatencyMs * a.runCount;
       }
     }
 
     const avgLatencyMs =
-      runCount > 0 ? Math.round(totalWeightedLatency / runCount) : undefined;
+      runCount > 0 ? Math.round(latencySumMs / runCount) : undefined;
 
     // Min/max across all hourly buckets
     const minValues = bucket.aggregates
@@ -341,10 +369,16 @@ async function rollupHourlyAggregates(params: RollupParams) {
     const maxValues = bucket.aggregates
       .map((a) => a.maxLatencyMs)
       .filter((v): v is number => v !== null);
+    const p95Values = bucket.aggregates
+      .map((a) => a.p95LatencyMs)
+      .filter((v): v is number => v !== null);
     const minLatencyMs =
       minValues.length > 0 ? Math.min(...minValues) : undefined;
     const maxLatencyMs =
       maxValues.length > 0 ? Math.max(...maxValues) : undefined;
+    // Use max of hourly p95s as upper bound approximation
+    const p95LatencyMs =
+      p95Values.length > 0 ? Math.max(...p95Values) : undefined;
 
     // Insert daily aggregate
     await db.insert(healthCheckAggregates).values({
@@ -356,10 +390,11 @@ async function rollupHourlyAggregates(params: RollupParams) {
       healthyCount,
       degradedCount,
       unhealthyCount,
+      latencySumMs: latencySumMs > 0 ? latencySumMs : undefined,
       avgLatencyMs,
       minLatencyMs,
       maxLatencyMs,
-      p95LatencyMs: undefined, // Cannot accurately combine p95s
+      p95LatencyMs,
       aggregatedResult: undefined, // Cannot combine result across hours
     });
 
@@ -399,9 +434,3 @@ async function deleteExpiredAggregates(params: DeleteExpiredParams) {
       ),
     );
 }
-
-function calculatePercentile(values: number[], percentile: number): number {
-  const sorted = values.toSorted((a, b) => a - b);
-  const index = Math.ceil((percentile / 100) * sorted.length) - 1;
-  return sorted[Math.max(0, index)];
-}

package/src/router.test.ts

@@ -43,7 +43,18 @@ describe("HealthCheck Router", () => {
     getStrategiesWithMeta: mock(() => []),
   };
 
-  const router = createHealthCheckRouter(mockDb as never, mockRegistry);
+  const mockCollectorRegistry = {
+    register: mock(),
+    getCollector: mock(),
+    getCollectors: mock(() => []),
+    getCollectorsForPlugin: mock(() => []),
+  };
+
+  const router = createHealthCheckRouter(
+    mockDb as never,
+    mockRegistry,
+    mockCollectorRegistry as never,
+  );
 
   it("getStrategies returns strategies from registry", async () => {
     const context = createMockRpcContext({
@@ -119,7 +130,7 @@ describe("HealthCheck Router", () => {
     const result = await call(
       router.getCollectors,
       { strategyId: "healthcheck-ssh" },
-      { context }
+      { context },
     );
     expect(result).toHaveLength(1);
     expect(result[0].id).toBe("collector-hardware.cpu");
@@ -139,7 +150,7 @@ describe("HealthCheck Router", () => {
     const result = await call(
       router.getCollectors,
       { strategyId: "unknown" },
-      { context }
+      { context },
     );
     expect(result).toHaveLength(0);
   });

package/src/router.ts

@@ -5,6 +5,7 @@ import {
   type RpcContext,
   type HealthCheckRegistry,
   type SafeDatabase,
+  type CollectorRegistry,
 } from "@checkstack/backend-api";
 import { healthCheckContract } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
@@ -20,9 +21,10 @@ import { toJsonSchemaWithChartMeta } from "./schema-utils";
 export const createHealthCheckRouter = (
   database: SafeDatabase<typeof schema>,
   registry: HealthCheckRegistry,
+  collectorRegistry: CollectorRegistry,
 ) => {
   // Create service instance once - shared across all handlers
-  const service = new HealthCheckService(database, registry);
+  const service = new HealthCheckService(database, registry, collectorRegistry);
 
   // Create contract implementer with context type AND auto auth middleware
   const os = implement(healthCheckContract)
@@ -74,6 +76,9 @@ export const createHealthCheckRouter = (
         description: collector.description,
         configSchema: toJsonSchema(collector.config.schema),
         resultSchema: toJsonSchemaWithChartMeta(collector.result.schema),
+        aggregatedResultSchema: collector.aggregatedResult
+          ? toJsonSchemaWithChartMeta(collector.aggregatedResult.schema)
+          : undefined,
         allowMultiple: collector.allowMultiple ?? false,
       }));
     }),
@@ -100,6 +105,14 @@ export const createHealthCheckRouter = (
       await service.deleteConfiguration(input);
     }),
 
+    pauseConfiguration: os.pauseConfiguration.handler(async ({ input }) => {
+      await service.pauseConfiguration(input);
+    }),
+
+    resumeConfiguration: os.resumeConfiguration.handler(async ({ input }) => {
+      await service.resumeConfiguration(input);
+    }),
+
     getSystemConfigurations: os.getSystemConfigurations.handler(
       async ({ input }) => {
         return service.getSystemConfigurations(input);

package/src/schema.ts

@@ -45,9 +45,11 @@ export const healthCheckConfigurations = pgTable(
     collectors: jsonb("collectors").$type<CollectorConfigEntry[]>(),
     intervalSeconds: integer("interval_seconds").notNull(),
     isTemplate: boolean("is_template").default(false),
+    /** Whether this configuration is paused (execution skipped for all systems) */
+    paused: boolean("paused").default(false).notNull(),
     createdAt: timestamp("created_at").defaultNow().notNull(),
     updatedAt: timestamp("updated_at").defaultNow().notNull(),
-  }
+  },
 );
 
 /**
@@ -93,7 +95,7 @@ export const systemHealthChecks = pgTable(
   },
   (t) => ({
     pk: primaryKey({ columns: [t.systemId, t.configurationId] }),
-  })
+  }),
 );
 
 export const healthCheckRuns = pgTable("health_check_runs", {
@@ -137,6 +139,8 @@ export const healthCheckAggregates = pgTable(
     healthyCount: integer("healthy_count").notNull(),
     degradedCount: integer("degraded_count").notNull(),
     unhealthyCount: integer("unhealthy_count").notNull(),
+    /** Sum of all latencies in this bucket (for accurate averaging when combining) */
+    latencySumMs: integer("latency_sum_ms"),
     avgLatencyMs: integer("avg_latency_ms"),
     minLatencyMs: integer("min_latency_ms"),
     maxLatencyMs: integer("max_latency_ms"),
@@ -152,7 +156,7 @@ export const healthCheckAggregates = pgTable(
       t.configurationId,
       t.systemId,
       t.bucketStart,
-      t.bucketSize
+      t.bucketSize,
     ),
-  })
+  }),
 );

package/src/service-pause.test.ts

@@ -0,0 +1,50 @@
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+import { HealthCheckService } from "./service";
+import { createMockDb } from "@checkstack/test-utils-backend";
+
+describe("HealthCheckService - pause/resume", () => {
+  let mockDb: ReturnType<typeof createMockDb>;
+  let service: HealthCheckService;
+  let mockUpdate: ReturnType<typeof mock>;
+  let mockSet: ReturnType<typeof mock>;
+  let mockWhere: ReturnType<typeof mock>;
+
+  beforeEach(() => {
+    mockDb = createMockDb();
+    mockWhere = mock(() => Promise.resolve());
+    mockSet = mock(() => ({ where: mockWhere }));
+    mockUpdate = mock(() => ({ set: mockSet }));
+    (mockDb.update as any) = mockUpdate;
+    service = new HealthCheckService(mockDb as any);
+  });
+
+  describe("pauseConfiguration", () => {
+    it("should update paused to true and set updatedAt", async () => {
+      await service.pauseConfiguration("config-123");
+
+      expect(mockUpdate).toHaveBeenCalled();
+      expect(mockSet).toHaveBeenCalledWith(
+        expect.objectContaining({
+          paused: true,
+          updatedAt: expect.any(Date),
+        }),
+      );
+      expect(mockWhere).toHaveBeenCalled();
+    });
+  });
+
+  describe("resumeConfiguration", () => {
+    it("should update paused to false and set updatedAt", async () => {
+      await service.resumeConfiguration("config-456");
+
+      expect(mockUpdate).toHaveBeenCalled();
+      expect(mockSet).toHaveBeenCalledWith(
+        expect.objectContaining({
+          paused: false,
+          updatedAt: expect.any(Date),
+        }),
+      );
+      expect(mockWhere).toHaveBeenCalled();
+    });
+  });
+});