@checkstack/healthcheck-backend 1.1.3 → 1.2.0

package/src/schema.ts

@@ -9,10 +9,12 @@ import {
   timestamp,
   primaryKey,
   unique,
+  index,
 } from "drizzle-orm/pg-core";
 import type {
   StateThresholds,
   CollectorConfigEntry,
+  NotificationPolicy,
 } from "@checkstack/healthcheck-common";
 import type { VersionedRecord } from "@checkstack/backend-api";
 
@@ -100,6 +102,12 @@ export const systemHealthChecks = pgTable(
      * Defaults to true. Only relevant when satelliteIds is set.
      */
     includeLocal: boolean("include_local").default(true).notNull(),
+    /**
+     * Per-association notification policy. Null falls back to platform
+     * defaults (no suppression).
+     */
+    notificationPolicy:
+      jsonb("notification_policy").$type<NotificationPolicy>(),
     createdAt: timestamp("created_at").defaultNow().notNull(),
     updatedAt: timestamp("updated_at").defaultNow().notNull(),
   },
@@ -108,6 +116,74 @@ export const systemHealthChecks = pgTable(
   }),
 );
 
+/**
+ * Records each time a check's *evaluated* state transitions from
+ * non-unhealthy to unhealthy. Used to decide whether the per-check
+ * incident threshold (N transitions in M minutes) has been met.
+ * Pruned by the retention job alongside raw runs.
+ */
+export const healthCheckUnhealthyTransitions = pgTable(
+  "health_check_unhealthy_transitions",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    configurationId: uuid("configuration_id")
+      .notNull()
+      .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
+    systemId: text("system_id").notNull(),
+    transitionedAt: timestamp("transitioned_at").defaultNow().notNull(),
+  },
+  (t) => ({
+    // Powers the threshold count query
+    // (WHERE config_id = ? AND system_id = ? AND transitioned_at > ?).
+    lookupIdx: index(
+      "health_check_unhealthy_transitions_lookup_idx",
+    ).on(t.configurationId, t.systemId, t.transitionedAt),
+  }),
+);
+
+/**
+ * Mapping of auto-opened incidents back to the system + check that
+ * triggered them. `closedAt` stays null while the incident is active;
+ * the auto-close worker sets it once the linked system has been
+ * steadily healthy for the cooldown.
+ *
+ * No FK to the incident table — that lives in another plugin's schema
+ * and we treat it as a soft reference (incident deletes are handled
+ * by the auto-close worker, which tolerates missing rows).
+ */
+export const healthCheckAutoIncidents = pgTable(
+  "health_check_auto_incidents",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    incidentId: uuid("incident_id").notNull(),
+    systemId: text("system_id").notNull(),
+    configurationId: uuid("configuration_id")
+      .notNull()
+      .references(() => healthCheckConfigurations.id, { onDelete: "cascade" }),
+    openedAt: timestamp("opened_at").defaultNow().notNull(),
+    closedAt: timestamp("closed_at"),
+    /**
+     * Auto-close cooldown snapshot taken when the incident was opened.
+     * `null` means "never auto-close" — the worker leaves this
+     * incident alone and an operator must resolve it manually. Stored
+     * per-row so a later policy change doesn't retroactively alter
+     * the close behaviour of incidents already in flight.
+     */
+    cooldownMinutes: integer("cooldown_minutes"),
+  },
+  (t) => ({
+    // Powers "is there an active auto-incident for this system?" check.
+    activeBySystemIdx: index(
+      "health_check_auto_incidents_active_by_system_idx",
+    ).on(t.systemId, t.closedAt),
+    // Powers "find the most recent close for this assignment" lookup
+    // used by the require-recovery-before-reopen check.
+    lastCloseByAssignmentIdx: index(
+      "health_check_auto_incidents_last_close_idx",
+    ).on(t.configurationId, t.systemId, t.closedAt),
+  }),
+);
+
 export const healthCheckRuns = pgTable("health_check_runs", {
   id: uuid("id").primaryKey().defaultRandom(),
   configurationId: uuid("configuration_id")

package/src/service-notification-policy.test.ts

@@ -0,0 +1,174 @@
+import { describe, it, expect, mock } from "bun:test";
+import { HealthCheckService } from "./service";
+import { createMockDb } from "@checkstack/test-utils-backend";
+import {
+  DEFAULT_NOTIFICATION_POLICY,
+  type NotificationPolicy,
+} from "@checkstack/healthcheck-common";
+
+/**
+ * Build a service whose only DB interaction is the chain used by
+ * `getAssignmentNotificationPolicy`. The chain ends in `.limit(1)` and
+ * returns the supplied rows verbatim. An optional in-memory platform
+ * default stands in for the ConfigService.
+ */
+function buildServiceWithRows(
+  rows: unknown[],
+  platformDefault?: NotificationPolicy,
+): HealthCheckService {
+  const mockDb = createMockDb();
+  const limitChain = mock(async () => rows);
+  const whereChain = mock(() => ({ limit: limitChain }));
+  const fromChain = mock(() => ({ where: whereChain }));
+  const selectChain = mock(() => ({ from: fromChain }));
+  (mockDb as { select: unknown }).select = selectChain;
+
+  const configService =
+    platformDefault === undefined
+      ? undefined
+      : ({
+          get: mock(async () => platformDefault),
+          set: mock(async () => {}),
+        } as never);
+
+  return new HealthCheckService(
+    mockDb as never,
+    {} as never,
+    {} as never,
+    configService,
+  );
+}
+
+describe("HealthCheckService.getAssignmentNotificationPolicy", () => {
+  it("falls back to compile-time defaults when no association and no platform defaults", async () => {
+    const service = buildServiceWithRows([]);
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy).toEqual(DEFAULT_NOTIFICATION_POLICY);
+  });
+
+  it("falls back to platform defaults when association exists but notificationPolicy is null", async () => {
+    const customPlatformDefault: NotificationPolicy = {
+      ...DEFAULT_NOTIFICATION_POLICY,
+      autoCloseAfterMinutes: 120,
+      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 15 },
+    };
+    const service = buildServiceWithRows(
+      [{ notificationPolicy: null }],
+      customPlatformDefault,
+    );
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy.autoCloseAfterMinutes).toBe(120);
+    expect(policy.sustainedUnhealthyTrigger.durationMinutes).toBe(15);
+  });
+
+  it("falls back to platform defaults when no association exists", async () => {
+    const customPlatformDefault: NotificationPolicy = {
+      ...DEFAULT_NOTIFICATION_POLICY,
+      flappingTrigger: { enabled: true, transitions: 10, windowMinutes: 30 },
+    };
+    const service = buildServiceWithRows([], customPlatformDefault);
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy.flappingTrigger).toEqual({
+      enabled: true,
+      transitions: 10,
+      windowMinutes: 30,
+    });
+  });
+
+  it("prefers per-assignment override over platform defaults", async () => {
+    const platformDefault: NotificationPolicy = {
+      ...DEFAULT_NOTIFICATION_POLICY,
+      autoOpenIncidentOnUnhealthy: false,
+    };
+    const assignmentOverride = {
+      suppressDeEscalations: true,
+      autoOpenIncidentOnUnhealthy: true, // overrides platform default
+      useNotificationSuppression: true,
+      skipDuringMaintenance: true,
+      sustainedUnhealthyTrigger: { enabled: true, durationMinutes: 30 },
+      flappingTrigger: { enabled: true, transitions: 3, windowMinutes: 60 },
+      autoCloseAfterMinutes: 30,
+    };
+    const service = buildServiceWithRows(
+      [{ notificationPolicy: assignmentOverride }],
+      platformDefault,
+    );
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy.autoOpenIncidentOnUnhealthy).toBe(true);
+    expect(policy.suppressDeEscalations).toBe(true);
+  });
+
+  it("fills in defaults for partial stored policies", async () => {
+    // Older rows may have only `suppressDeEscalations` set from the
+    // first migration. All other fields must default in.
+    const service = buildServiceWithRows([
+      { notificationPolicy: { suppressDeEscalations: true } },
+    ]);
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy.suppressDeEscalations).toBe(true);
+    expect(policy.autoOpenIncidentOnUnhealthy).toBe(true);
+    expect(policy.useNotificationSuppression).toBe(true);
+    expect(policy.skipDuringMaintenance).toBe(true);
+    expect(policy.sustainedUnhealthyTrigger).toEqual({
+      enabled: true,
+      durationMinutes: 30,
+    });
+    expect(policy.flappingTrigger).toEqual({
+      enabled: true,
+      transitions: 3,
+      windowMinutes: 60,
+    });
+    expect(policy.autoCloseAfterMinutes).toBe(30);
+  });
+
+  it("returns explicit values exactly when fully specified", async () => {
+    const service = buildServiceWithRows([
+      {
+        notificationPolicy: {
+          suppressDeEscalations: false,
+          autoOpenIncidentOnUnhealthy: false,
+          useNotificationSuppression: false,
+          skipDuringMaintenance: false,
+          sustainedUnhealthyTrigger: { enabled: false, durationMinutes: 15 },
+          flappingTrigger: {
+            enabled: true,
+            transitions: 5,
+            windowMinutes: 30,
+          },
+          autoCloseAfterMinutes: null,
+        },
+      },
+    ]);
+    const policy = await service.getAssignmentNotificationPolicy({
+      systemId: "sys-1",
+      configurationId: "cfg-1",
+    });
+    expect(policy.autoOpenIncidentOnUnhealthy).toBe(false);
+    expect(policy.skipDuringMaintenance).toBe(false);
+    expect(policy.sustainedUnhealthyTrigger).toEqual({
+      enabled: false,
+      durationMinutes: 15,
+    });
+    expect(policy.flappingTrigger).toEqual({
+      enabled: true,
+      transitions: 5,
+      windowMinutes: 30,
+    });
+    expect(policy.autoCloseAfterMinutes).toBeNull();
+  });
+});

package/src/service.ts

@@ -6,7 +6,16 @@ import {
   HealthCheckStatus,
   RetentionConfig,
   type HealthCheckRunResult,
+  type NotificationPolicy,
+  NotificationPolicySchema,
+  DEFAULT_NOTIFICATION_POLICY,
 } from "@checkstack/healthcheck-common";
+import type { ConfigService } from "@checkstack/backend-api";
+import {
+  notificationDefaultsConfigV1,
+  NOTIFICATION_DEFAULTS_CONFIG_ID,
+  NOTIFICATION_DEFAULTS_CONFIG_VERSION,
+} from "./notification-defaults-config";
 import {
   healthCheckConfigurations,
   systemHealthChecks,
@@ -15,7 +24,16 @@ import {
   VersionedStateThresholds,
 } from "./schema";
 import * as schema from "./schema";
-import { eq, and, InferSelectModel, desc, gte, lte, isNull } from "drizzle-orm";
+import {
+  eq,
+  and,
+  InferSelectModel,
+  desc,
+  gte,
+  lte,
+  isNull,
+  inArray,
+} from "drizzle-orm";
 import { ORPCError } from "@orpc/server";
 import { evaluateHealthStatus } from "./state-evaluator";
 import { stateThresholds } from "./state-thresholds-migrations";
@@ -57,8 +75,56 @@ export class HealthCheckService {
     private db: Db,
     private registry: HealthCheckRegistry,
     private collectorRegistry: CollectorRegistry,
+    /**
+     * Optional — only required by code paths that resolve platform
+     * defaults (notification policy fallback). When absent, callers
+     * fall back to the compile-time `DEFAULT_NOTIFICATION_POLICY`.
+     * Kept optional so existing GitOps-only / test constructions don't
+     * have to plumb it through.
+     */
+    private configService?: ConfigService,
   ) {}
 
+  /**
+   * Resolve the platform-wide notification policy defaults. Returns
+   * the compile-time defaults when no `configService` was provided or
+   * nothing has ever been persisted. Stored values are passed through
+   * the schema so missing fields default in.
+   */
+  async getPlatformNotificationDefaults(): Promise<NotificationPolicy> {
+    if (!this.configService) {
+      return DEFAULT_NOTIFICATION_POLICY;
+    }
+    const stored = await this.configService.get(
+      NOTIFICATION_DEFAULTS_CONFIG_ID,
+      notificationDefaultsConfigV1,
+      NOTIFICATION_DEFAULTS_CONFIG_VERSION,
+    );
+    return stored ?? DEFAULT_NOTIFICATION_POLICY;
+  }
+
+  /**
+   * Persist platform-wide notification policy defaults. Per-assignment
+   * rows with `notificationPolicy = null` will read the new defaults
+   * on their next evaluation. In-flight auto-incidents are unaffected
+   * (their cooldown is snapshotted per-row at open time).
+   */
+  async setPlatformNotificationDefaults(
+    policy: NotificationPolicy,
+  ): Promise<void> {
+    if (!this.configService) {
+      throw new Error(
+        "ConfigService not configured; cannot persist platform notification defaults",
+      );
+    }
+    await this.configService.set(
+      NOTIFICATION_DEFAULTS_CONFIG_ID,
+      notificationDefaultsConfigV1,
+      NOTIFICATION_DEFAULTS_CONFIG_VERSION,
+      policy,
+    );
+  }
+
   async createConfiguration(
     data: CreateHealthCheckConfiguration,
   ): Promise<HealthCheckConfiguration> {
@@ -133,6 +199,7 @@ export class HealthCheckService {
     stateThresholds?: StateThresholds;
     satelliteIds?: string[];
     includeLocal?: boolean;
+    notificationPolicy?: NotificationPolicy;
   }) {
     const {
       systemId,
@@ -141,6 +208,7 @@ export class HealthCheckService {
       stateThresholds: stateThresholds_,
       satelliteIds,
       includeLocal = true,
+      notificationPolicy,
     } = props;
 
     // Wrap thresholds in versioned config if provided
@@ -156,6 +224,7 @@ export class HealthCheckService {
         stateThresholds: versionedThresholds,
         satelliteIds: satelliteIds ?? undefined,
         includeLocal,
+        notificationPolicy: notificationPolicy ?? undefined,
       })
       .onConflictDoUpdate({
         target: [
@@ -167,6 +236,7 @@ export class HealthCheckService {
           stateThresholds: versionedThresholds,
           satelliteIds: satelliteIds ?? undefined,
           includeLocal,
+          notificationPolicy: notificationPolicy ?? undefined,
           updatedAt: new Date(),
         },
       });
@@ -282,6 +352,7 @@ export class HealthCheckService {
         stateThresholds: systemHealthChecks.stateThresholds,
         satelliteIds: systemHealthChecks.satelliteIds,
         includeLocal: systemHealthChecks.includeLocal,
+        notificationPolicy: systemHealthChecks.notificationPolicy,
       })
       .from(systemHealthChecks)
       .innerJoin(
@@ -304,11 +375,55 @@ export class HealthCheckService {
         stateThresholds: thresholds,
         satelliteIds: row.satelliteIds ?? undefined,
         includeLocal: row.includeLocal,
+        notificationPolicy: row.notificationPolicy ?? undefined,
       });
     }
     return results;
   }
 
+  /**
+   * Resolve the fully-defaulted notification policy for a single
+   * (system, configuration) association. Resolution order:
+   *
+   *   1. Per-assignment override (`systemHealthChecks.notificationPolicy`)
+   *      when non-null. Stored as a full policy; missing keys defaulted
+   *      via zod parse.
+   *   2. Platform-wide defaults via `ConfigService`.
+   *   3. Compile-time `DEFAULT_NOTIFICATION_POLICY`.
+   *
+   * The all-or-nothing semantic is intentional: assignment rows are
+   * either fully-overridden or fully-inherited from the platform.
+   * Operators can revert an override by setting the row's policy to
+   * `null`, which is the "Use platform defaults" action in the UI.
+   */
+  async getAssignmentNotificationPolicy({
+    systemId,
+    configurationId,
+  }: {
+    systemId: string;
+    configurationId: string;
+  }): Promise<NotificationPolicy> {
+    const [row] = await this.db
+      .select({
+        notificationPolicy: systemHealthChecks.notificationPolicy,
+      })
+      .from(systemHealthChecks)
+      .where(
+        and(
+          eq(systemHealthChecks.systemId, systemId),
+          eq(systemHealthChecks.configurationId, configurationId),
+        ),
+      )
+      .limit(1);
+
+    // No assignment row → use platform defaults (the only sensible
+    // value for a configuration nothing has explicitly touched).
+    if (!row || row.notificationPolicy === null) {
+      return this.getPlatformNotificationDefaults();
+    }
+    return NotificationPolicySchema.parse(row.notificationPolicy);
+  }
+
   /**
    * Get the evaluated health status for a system based on configured thresholds.
    * Aggregates status from all health check configurations for this system.
@@ -489,6 +604,7 @@ export class HealthCheckService {
     startDate?: Date;
     endDate?: Date;
     sourceFilter?: string;
+    statusFilter?: HealthCheckStatus[];
     limit?: number;
     offset?: number;
     sortOrder: "asc" | "desc";
@@ -499,6 +615,7 @@ export class HealthCheckService {
       startDate,
       endDate,
       sourceFilter,
+      statusFilter,
       limit = 10,
       offset = 0,
       sortOrder,
@@ -518,6 +635,11 @@ export class HealthCheckService {
       conditions.push(eq(healthCheckRuns.sourceId, sourceFilter));
     }
 
+    // Status filtering (e.g. only failing runs)
+    if (statusFilter && statusFilter.length > 0) {
+      conditions.push(inArray(healthCheckRuns.status, statusFilter));
+    }
+
     // Build where clause
     const whereClause = conditions.length > 0 ? and(...conditions) : undefined;
 
@@ -563,6 +685,7 @@ export class HealthCheckService {
     startDate?: Date;
     endDate?: Date;
     sourceFilter?: string;
+    statusFilter?: HealthCheckStatus[];
     limit?: number;
     offset?: number;
     sortOrder: "asc" | "desc";
@@ -573,6 +696,7 @@ export class HealthCheckService {
       startDate,
       endDate,
       sourceFilter,
+      statusFilter,
       limit = 10,
       offset = 0,
       sortOrder,
@@ -592,6 +716,11 @@ export class HealthCheckService {
       conditions.push(eq(healthCheckRuns.sourceId, sourceFilter));
     }
 
+    // Status filtering (e.g. only failing runs)
+    if (statusFilter && statusFilter.length > 0) {
+      conditions.push(inArray(healthCheckRuns.status, statusFilter));
+    }
+
     const whereClause = conditions.length > 0 ? and(...conditions) : undefined;
     const total = await this.db.$count(healthCheckRuns, whereClause);
 

package/tsconfig.json

@@ -37,6 +37,9 @@
     {
       "path": "../healthcheck-common"
     },
+    {
+      "path": "../incident-backend"
+    },
     {
       "path": "../incident-common"
     },