@checkstack/healthcheck-backend 0.4.1 → 0.5.0

package/src/retention-job.ts

@@ -1,23 +1,32 @@
-import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type {
+  HealthCheckRegistry,
+  Logger,
+  SafeDatabase,
+  CollectorRegistry,
+} from "@checkstack/backend-api";
 import * as schema from "./schema";
 import {
   healthCheckRuns,
   systemHealthChecks,
   healthCheckAggregates,
+  healthCheckConfigurations,
   DEFAULT_RETENTION_CONFIG,
 } from "./schema";
 import { eq, and, lt, sql } from "drizzle-orm";
-import type {
-  HealthCheckRegistry,
-  Logger,
-} from "@checkstack/backend-api";
 import type { QueueManager } from "@checkstack/queue-api";
+import {
+  aggregateCollectorData,
+  calculateLatencyStats,
+  countStatuses,
+  extractLatencies,
+} from "./aggregation-utils";
 
-type Db = NodePgDatabase<typeof schema>;
+type Db = SafeDatabase<typeof schema>;
 
 interface RetentionJobDeps {
   db: Db;
   registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
   logger: Logger;
   queueManager: QueueManager;
 }
@@ -35,7 +44,7 @@ interface RetentionJobPayload {
  * 3. Deletes expired daily aggregates
  */
 export async function setupRetentionJob(deps: RetentionJobDeps) {
-  const { queueManager, logger, db, registry } = deps;
+  const { queueManager, logger, db, registry, collectorRegistry } = deps;
 
   const queue = queueManager.getQueue<RetentionJobPayload>(RETENTION_QUEUE);
 
@@ -43,10 +52,16 @@ export async function setupRetentionJob(deps: RetentionJobDeps) {
   await queue.consume(
     async () => {
       logger.info("Starting health check retention job");
-      await runRetentionJob({ db, registry, logger, queueManager });
+      await runRetentionJob({
+        db,
+        registry,
+        collectorRegistry,
+        logger,
+        queueManager,
+      });
       logger.info("Completed health check retention job");
     },
-    { consumerGroup: "retention-worker" }
+    { consumerGroup: "retention-worker" },
   );
 
   // Schedule daily retention run (86400 seconds = 24 hours)
@@ -55,7 +70,7 @@ export async function setupRetentionJob(deps: RetentionJobDeps) {
     {
       jobId: "health-check-retention-daily",
       intervalSeconds: 24 * 60 * 60, // Daily (24 hours)
-    }
+    },
   );
 
   logger.info("Health check retention job scheduled (runs daily)");
@@ -65,7 +80,7 @@ export async function setupRetentionJob(deps: RetentionJobDeps) {
  * Main retention job logic
  */
 export async function runRetentionJob(deps: RetentionJobDeps) {
-  const { db, registry, logger } = deps;
+  const { db, registry, collectorRegistry, logger } = deps;
 
   // Get all unique system-config assignments
   const assignments = await db.select().from(systemHealthChecks);
@@ -79,6 +94,7 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
       await aggregateRawRuns({
         db,
         registry,
+        collectorRegistry,
         systemId: assignment.systemId,
         configurationId: assignment.configurationId,
         rawRetentionDays: retentionConfig.rawRetentionDays,
@@ -102,7 +118,7 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
     } catch (error) {
       logger.error(
         `Retention job failed for ${assignment.systemId}/${assignment.configurationId}`,
-        { error }
+        { error },
       );
     }
   }
@@ -111,6 +127,7 @@ export async function runRetentionJob(deps: RetentionJobDeps) {
 interface AggregateRawRunsParams {
   db: Db;
   registry: HealthCheckRegistry;
+  collectorRegistry: CollectorRegistry;
   systemId: string;
   configurationId: string;
   rawRetentionDays: number;
@@ -120,16 +137,25 @@ interface AggregateRawRunsParams {
  * Aggregates raw runs older than retention period into hourly buckets
  */
 async function aggregateRawRuns(params: AggregateRawRunsParams) {
-  const { db, registry, systemId, configurationId, rawRetentionDays } = params;
+  const {
+    db,
+    registry,
+    collectorRegistry,
+    systemId,
+    configurationId,
+    rawRetentionDays,
+  } = params;
 
   const cutoffDate = new Date();
   cutoffDate.setDate(cutoffDate.getDate() - rawRetentionDays);
   cutoffDate.setHours(cutoffDate.getHours(), 0, 0, 0); // Round to hour
 
   // Get strategy for metadata aggregation
-  const config = await db.query.healthCheckConfigurations.findFirst({
-    where: eq(schema.healthCheckConfigurations.id, configurationId),
-  });
+  const [config] = await db
+    .select()
+    .from(healthCheckConfigurations)
+    .where(eq(healthCheckConfigurations.id, configurationId))
+    .limit(1);
   const strategy = config ? registry.getStrategy(config.strategyId) : undefined;
 
   // Query raw runs older than cutoff, grouped by hour
@@ -140,8 +166,8 @@ async function aggregateRawRuns(params: AggregateRawRunsParams) {
       and(
         eq(healthCheckRuns.systemId, systemId),
         eq(healthCheckRuns.configurationId, configurationId),
-        lt(healthCheckRuns.timestamp, cutoffDate)
-      )
+        lt(healthCheckRuns.timestamp, cutoffDate),
+      ),
     )
     .orderBy(healthCheckRuns.timestamp);
 
@@ -180,42 +206,43 @@ async function aggregateRawRuns(params: AggregateRawRunsParams) {
 
   // Create aggregates and delete raw runs
   for (const [, bucket] of buckets) {
-    // Calculate metrics
     const runCount = bucket.runs.length;
-    let healthyCount = 0;
-    let degradedCount = 0;
-    let unhealthyCount = 0;
-    for (const r of bucket.runs) {
-      if (r.status === "healthy") healthyCount++;
-      if (r.status === "degraded") degradedCount++;
-      if (r.status === "unhealthy") unhealthyCount++;
-    }
 
-    const latencies = bucket.runs
-      .map((r) => r.latencyMs)
-      .filter((l): l is number => l !== undefined);
-
-    let avgLatencyMs: number | undefined;
-    let minLatencyMs: number | undefined;
-    let maxLatencyMs: number | undefined;
-    let p95LatencyMs: number | undefined;
-
-    if (latencies.length > 0) {
-      let sum = 0;
-      for (const l of latencies) sum += l;
-      avgLatencyMs = Math.round(sum / latencies.length);
-      minLatencyMs = Math.min(...latencies);
-      maxLatencyMs = Math.max(...latencies);
-      p95LatencyMs = calculatePercentile(latencies, 95);
-    }
+    // Calculate status counts
+    const { healthyCount, degradedCount, unhealthyCount } = countStatuses(
+      bucket.runs,
+    );
 
-    // Aggregate result if strategy is available
+    // Calculate latency stats
+    const latencies = extractLatencies(bucket.runs);
+    const {
+      latencySumMs,
+      avgLatencyMs,
+      minLatencyMs,
+      maxLatencyMs,
+      p95LatencyMs,
+    } = calculateLatencyStats(latencies);
+
+    // Aggregate strategy result
     let aggregatedResult: Record<string, unknown> | undefined;
     if (strategy) {
-      aggregatedResult = strategy.aggregateResult(bucket.runs) as Record<
+      const strategyResult = strategy.aggregateResult(bucket.runs) as Record<
         string,
         unknown
       >;
+
+      // Aggregate collector data
+      const collectorsAggregated = aggregateCollectorData(
+        bucket.runs,
+        collectorRegistry,
+      );
+
+      aggregatedResult = {
+        ...strategyResult,
+        ...(Object.keys(collectorsAggregated).length > 0
+          ? { collectors: collectorsAggregated }
+          : {}),
+      };
     }
 
     // Insert or update aggregate
@@ -230,6 +257,7 @@ async function aggregateRawRuns(params: AggregateRawRunsParams) {
         healthyCount,
         degradedCount,
         unhealthyCount,
+        latencySumMs,
         avgLatencyMs,
         minLatencyMs,
         maxLatencyMs,
@@ -284,8 +312,8 @@ async function rollupHourlyAggregates(params: RollupParams) {
         eq(healthCheckAggregates.systemId, systemId),
         eq(healthCheckAggregates.configurationId, configurationId),
         eq(healthCheckAggregates.bucketSize, "hourly"),
-        lt(healthCheckAggregates.bucketStart, cutoffDate)
-      )
+        lt(healthCheckAggregates.bucketStart, cutoffDate),
+      ),
     );
 
   if (oldHourly.length === 0) return;
@@ -316,20 +344,23 @@ async function rollupHourlyAggregates(params: RollupParams) {
     let healthyCount = 0;
     let degradedCount = 0;
     let unhealthyCount = 0;
-    let totalWeightedLatency = 0;
+    let latencySumMs = 0;
 
     for (const a of bucket.aggregates) {
       runCount += a.runCount;
       healthyCount += a.healthyCount;
       degradedCount += a.degradedCount;
       unhealthyCount += a.unhealthyCount;
-      if (a.avgLatencyMs !== null) {
-        totalWeightedLatency += a.avgLatencyMs * a.runCount;
+      // Use latencySumMs if available, fallback to avg*count approximation
+      if (a.latencySumMs !== null) {
+        latencySumMs += a.latencySumMs;
+      } else if (a.avgLatencyMs !== null) {
+        latencySumMs += a.avgLatencyMs * a.runCount;
       }
     }
 
     const avgLatencyMs =
-      runCount > 0 ? Math.round(totalWeightedLatency / runCount) : undefined;
+      runCount > 0 ? Math.round(latencySumMs / runCount) : undefined;
 
     // Min/max across all hourly buckets
     const minValues = bucket.aggregates
@@ -338,10 +369,16 @@ async function rollupHourlyAggregates(params: RollupParams) {
     const maxValues = bucket.aggregates
       .map((a) => a.maxLatencyMs)
       .filter((v): v is number => v !== null);
+    const p95Values = bucket.aggregates
+      .map((a) => a.p95LatencyMs)
+      .filter((v): v is number => v !== null);
     const minLatencyMs =
       minValues.length > 0 ? Math.min(...minValues) : undefined;
     const maxLatencyMs =
       maxValues.length > 0 ? Math.max(...maxValues) : undefined;
+    // Use max of hourly p95s as upper bound approximation
+    const p95LatencyMs =
+      p95Values.length > 0 ? Math.max(...p95Values) : undefined;
 
     // Insert daily aggregate
     await db.insert(healthCheckAggregates).values({
@@ -353,10 +390,11 @@ async function rollupHourlyAggregates(params: RollupParams) {
       healthyCount,
       degradedCount,
       unhealthyCount,
+      latencySumMs: latencySumMs > 0 ? latencySumMs : undefined,
       avgLatencyMs,
       minLatencyMs,
       maxLatencyMs,
-      p95LatencyMs: undefined, // Cannot accurately combine p95s
+      p95LatencyMs,
       aggregatedResult: undefined, // Cannot combine result across hours
     });
 
@@ -392,13 +430,7 @@ async function deleteExpiredAggregates(params: DeleteExpiredParams) {
         eq(healthCheckAggregates.systemId, systemId),
         eq(healthCheckAggregates.configurationId, configurationId),
         eq(healthCheckAggregates.bucketSize, "daily"),
-        lt(healthCheckAggregates.bucketStart, cutoffDate)
-      )
+        lt(healthCheckAggregates.bucketStart, cutoffDate),
+      ),
     );
 }
-
-function calculatePercentile(values: number[], percentile: number): number {
-  const sorted = values.toSorted((a, b) => a - b);
-  const index = Math.ceil((percentile / 100) * sorted.length) - 1;
-  return sorted[Math.max(0, index)];
-}

package/src/router.test.ts

@@ -43,7 +43,18 @@ describe("HealthCheck Router", () => {
     getStrategiesWithMeta: mock(() => []),
   };
 
-  const router = createHealthCheckRouter(mockDb as never, mockRegistry);
+  const mockCollectorRegistry = {
+    register: mock(),
+    getCollector: mock(),
+    getCollectors: mock(() => []),
+    getCollectorsForPlugin: mock(() => []),
+  };
+
+  const router = createHealthCheckRouter(
+    mockDb as never,
+    mockRegistry,
+    mockCollectorRegistry as never,
+  );
 
   it("getStrategies returns strategies from registry", async () => {
     const context = createMockRpcContext({
@@ -119,7 +130,7 @@ describe("HealthCheck Router", () => {
     const result = await call(
       router.getCollectors,
       { strategyId: "healthcheck-ssh" },
-      { context }
+      { context },
     );
     expect(result).toHaveLength(1);
     expect(result[0].id).toBe("collector-hardware.cpu");
@@ -139,7 +150,7 @@ describe("HealthCheck Router", () => {
     const result = await call(
       router.getCollectors,
       { strategyId: "unknown" },
-      { context }
+      { context },
     );
     expect(result).toHaveLength(0);
   });

package/src/router.ts

@@ -4,10 +4,11 @@ import {
   toJsonSchema,
   type RpcContext,
   type HealthCheckRegistry,
+  type SafeDatabase,
+  type CollectorRegistry,
 } from "@checkstack/backend-api";
 import { healthCheckContract } from "@checkstack/healthcheck-common";
 import { HealthCheckService } from "./service";
-import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import * as schema from "./schema";
 import { toJsonSchemaWithChartMeta } from "./schema-utils";
 
@@ -18,11 +19,12 @@ import { toJsonSchemaWithChartMeta } from "./schema-utils";
  * based on the contract's meta.userType and meta.access.
  */
 export const createHealthCheckRouter = (
-  database: NodePgDatabase<typeof schema>,
-  registry: HealthCheckRegistry
+  database: SafeDatabase<typeof schema>,
+  registry: HealthCheckRegistry,
+  collectorRegistry: CollectorRegistry,
 ) => {
   // Create service instance once - shared across all handlers
-  const service = new HealthCheckService(database, registry);
+  const service = new HealthCheckService(database, registry, collectorRegistry);
 
   // Create contract implementer with context type AND auto auth middleware
   const os = implement(healthCheckContract)
@@ -40,7 +42,7 @@ export const createHealthCheckRouter = (
           ? toJsonSchemaWithChartMeta(r.strategy.result.schema)
           : undefined,
         aggregatedResultSchema: toJsonSchemaWithChartMeta(
-          r.strategy.aggregatedResult.schema
+          r.strategy.aggregatedResult.schema,
         ),
       }));
     }),
@@ -48,7 +50,7 @@ export const createHealthCheckRouter = (
     getCollectors: os.getCollectors.handler(async ({ input, context }) => {
       // Get strategy to verify it exists
       const strategy = context.healthCheckRegistry.getStrategy(
-        input.strategyId
+        input.strategyId,
       );
       if (!strategy) {
         return [];
@@ -74,6 +76,9 @@ export const createHealthCheckRouter = (
         description: collector.description,
         configSchema: toJsonSchema(collector.config.schema),
         resultSchema: toJsonSchemaWithChartMeta(collector.result.schema),
+        aggregatedResultSchema: collector.aggregatedResult
+          ? toJsonSchemaWithChartMeta(collector.aggregatedResult.schema)
+          : undefined,
         allowMultiple: collector.allowMultiple ?? false,
       }));
     }),
@@ -103,13 +108,13 @@ export const createHealthCheckRouter = (
     getSystemConfigurations: os.getSystemConfigurations.handler(
       async ({ input }) => {
         return service.getSystemConfigurations(input);
-      }
+      },
     ),
 
     getSystemAssociations: os.getSystemAssociations.handler(
       async ({ input }) => {
         return service.getSystemAssociations(input.systemId);
-      }
+      },
     ),
 
     associateSystem: os.associateSystem.handler(async ({ input, context }) => {
@@ -123,7 +128,7 @@ export const createHealthCheckRouter = (
       // If enabling the health check, schedule it immediately
       if (input.body.enabled) {
         const config = await service.getConfiguration(
-          input.body.configurationId
+          input.body.configurationId,
         );
         if (config) {
           const { scheduleHealthCheck } = await import("./queue-executor");
@@ -152,9 +157,9 @@ export const createHealthCheckRouter = (
         await service.updateRetentionConfig(
           input.systemId,
           input.configurationId,
-          input.retentionConfig
+          input.retentionConfig,
         );
-      }
+      },
     ),
 
     getHistory: os.getHistory.handler(async ({ input }) => {
@@ -176,12 +181,12 @@ export const createHealthCheckRouter = (
         return service.getAggregatedHistory(input, {
           includeAggregatedResult: true,
         });
-      }
+      },
     ),
     getSystemHealthStatus: os.getSystemHealthStatus.handler(
       async ({ input }) => {
         return service.getSystemHealthStatus(input.systemId);
-      }
+      },
     ),
 
     getBulkSystemHealthStatus: os.getBulkSystemHealthStatus.handler(
@@ -195,17 +200,17 @@ export const createHealthCheckRouter = (
         await Promise.all(
           input.systemIds.map(async (systemId) => {
             statuses[systemId] = await service.getSystemHealthStatus(systemId);
-          })
+          }),
         );
 
         return { statuses };
-      }
+      },
     ),
 
     getSystemHealthOverview: os.getSystemHealthOverview.handler(
       async ({ input }) => {
         return service.getSystemHealthOverview(input.systemId);
-      }
+      },
     ),
   });
 };

package/src/schema.ts

@@ -47,7 +47,7 @@ export const healthCheckConfigurations = pgTable(
     isTemplate: boolean("is_template").default(false),
     createdAt: timestamp("created_at").defaultNow().notNull(),
     updatedAt: timestamp("updated_at").defaultNow().notNull(),
-  }
+  },
 );
 
 /**
@@ -93,7 +93,7 @@ export const systemHealthChecks = pgTable(
   },
   (t) => ({
     pk: primaryKey({ columns: [t.systemId, t.configurationId] }),
-  })
+  }),
 );
 
 export const healthCheckRuns = pgTable("health_check_runs", {
@@ -137,6 +137,8 @@ export const healthCheckAggregates = pgTable(
     healthyCount: integer("healthy_count").notNull(),
     degradedCount: integer("degraded_count").notNull(),
     unhealthyCount: integer("unhealthy_count").notNull(),
+    /** Sum of all latencies in this bucket (for accurate averaging when combining) */
+    latencySumMs: integer("latency_sum_ms"),
     avgLatencyMs: integer("avg_latency_ms"),
     minLatencyMs: integer("min_latency_ms"),
     maxLatencyMs: integer("max_latency_ms"),
@@ -152,7 +154,7 @@ export const healthCheckAggregates = pgTable(
       t.configurationId,
       t.systemId,
       t.bucketStart,
-      t.bucketSize
+      t.bucketSize,
     ),
-  })
+  }),
 );