@checkstack/healthcheck-common 1.1.2 → 1.2.0

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @checkstack/healthcheck-common
 
+## 1.2.0
+
+### Minor Changes
+
+- ba07ae2: Quiet down notification spam on flapping systems, auto-open incidents when a check goes critical, and let operators land directly on the broken checks.
+
+  Notification policy lives **per healthcheck assignment** (one row per `system × configuration`). Different checks on the same system are fully independent — disabling a setting on one check does not affect the others. Defaults preserve existing behaviour for `suppressDeEscalations`; **auto-incident defaults to on** for new and existing assignments.
+
+  - **`suppressDeEscalations`** (off by default). When on, transitions from a worse state to a better-but-still-failing state (e.g. `unhealthy → degraded`) no longer fire a notification. Escalations and full recoveries to `healthy` are unaffected. Resolved per assignment (the just-ran check is the one driving any aggregate transition).
+  - **`autoOpenIncidentOnUnhealthy`** (on by default). Either of two independent triggers can open the auto-incident:
+    - **`sustainedUnhealthyTrigger`** (default 30 min) — opens when the check stays continuously unhealthy for the configured duration. Catches real outages.
+    - **`flappingTrigger`** (default 3 transitions in 60 min) — opens when the check flips to unhealthy that many times in the window. Catches persistent flapping where each unhealthy phase is too brief for the sustained trigger.
+      Each trigger can be individually disabled. One incident per system: triggering checks attach to an existing active auto-incident.
+  - **`useNotificationSuppression`** (on by default, only meaningful when auto-open is on). Controls whether the auto-opened incident is created with `suppressNotifications: true` — leaving this off opens the incident but still pings operators on each transition.
+  - **`skipDuringMaintenance`** (on by default). No auto-incident is opened while the system has an active maintenance window with suppression. The system is intentionally down and shouldn't trip the on-call.
+  - **`autoCloseAfterMinutes`** (default 30). Auto-close cooldown is now per-assignment and snapshotted per-incident at open time — later policy edits don't alter in-flight incidents. Setting `null` ("Never auto-close") leaves the incident for manual resolution.
+  - **Require-recovery rule.** After any auto-incident closes (manual or auto), no new auto-incident can open until the check has logged at least one healthy run. Prevents a "operator dismissed but it's still broken" loop.
+  - **Auto-close worker** ticks every 60s and resolves auto-opened incidents whose systems have been healthy for their per-row `cooldownMinutes`. Rows with `null` cooldown are skipped entirely. Per-incident: failed close attempts are logged but never abort the sweep.
+  - **`incidentResolved` hook subscriber** syncs the auto-incident mapping when an operator manually resolves the incident, so the require-recovery rule sees the close immediately.
+  - **Platform-wide defaults.** New admin RPCs `getPlatformNotificationDefaults` / `setPlatformNotificationDefaults` (under the existing `healthcheck.configuration.{read,manage}` access rules) let operators set notification policy once for the whole instance. Per-assignment rows with `notificationPolicy: null` inherit the platform defaults at read time. UI: a "Notification defaults" button in the Assignment IDE opens a modal editor. The per-assignment Notifications panel shows an inheritance banner — "Using platform defaults" (read-only) with an "Override" button, or "Custom override" with a "Use platform defaults" button to revert. The all-or-nothing model keeps the mental model simple: each assignment is either fully inherited or fully overridden.
+  - **New service-level RPCs** on the incident plugin (`createAutoIncident`, `resolveAutoIncident`) let other plugins open/close incidents without a user context. Reused by the healthcheck auto-incident flow.
+  - **Health-state notification CTA** now deep-links to `?filter=failing` on the system detail page for non-recovery transitions (label changes to "View failing checks"). The system overview gains an `All / Failing / Healthy` segmented filter wired to the same `?filter=…` param.
+  - **Notification bell badge** now counts collapse groups instead of raw rows, so the number matches what the user sees in the notifications list. Built on `COUNT(DISTINCT COALESCE(collapse_key, id))` — notifications without a collapse key still each count as one.
+  - **`statusFilter` on `getHistory` / `getDetailedHistory`** lets the run-history page and the drawer's Recent Runs panel filter to `All / Healthy / Failing` via shared pills, with the page resetting to the first page on filter change.
+  - **Pagination defaults aligned with selector options.** Several pages defaulted to a page size (5 or 20) that wasn't in the dropdown's options (`[10, 25, 50, 100]`), so the page-size `<Select>` rendered empty. The drawer's Recent Runs now defaults to 10; the Run History, History List, and Delivery Logs pages now default to 25.
+
+  Includes Drizzle migrations adding the `notification_policy` jsonb column to `system_health_checks`, plus two new tables: `health_check_unhealthy_transitions` (for threshold counting) and `health_check_auto_incidents` (for mapping back to incident ids during auto-close).
+
 ## 1.1.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-common",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "license": "Elastic-2.0",
   "type": "module",
   "exports": {
@@ -9,16 +9,16 @@
     }
   },
   "dependencies": {
-    "@checkstack/common": "0.10.0",
-    "@checkstack/catalog-common": "2.2.1",
-    "@checkstack/notification-common": "1.1.1",
-    "@checkstack/signal-common": "0.2.3",
+    "@checkstack/common": "0.11.0",
+    "@checkstack/catalog-common": "2.2.2",
+    "@checkstack/notification-common": "1.2.0",
+    "@checkstack/signal-common": "0.2.4",
     "zod": "^4.2.1"
   },
   "devDependencies": {
     "typescript": "^5.7.2",
     "@checkstack/tsconfig": "0.0.7",
-    "@checkstack/scripts": "0.3.2"
+    "@checkstack/scripts": "0.3.3"
   },
   "scripts": {
     "typecheck": "tsgo -b",

package/src/rpc-contract.ts

@@ -17,6 +17,7 @@ import {
   RetentionConfigSchema,
   AggregatedBucketBaseSchema,
   AggregatedBucketSchema,
+  NotificationPolicySchema,
 } from "./schemas";
 
 // --- Response Schemas for Evaluated Status ---
@@ -155,6 +156,8 @@ export const healthCheckContract = {
           satelliteIds: z.array(z.string()).optional(),
           /** Whether to also run this check locally on the core (default: true) */
           includeLocal: z.boolean(),
+          /** Per-association notification policy (omitted = platform defaults) */
+          notificationPolicy: NotificationPolicySchema.optional(),
         }),
       ),
     ),
@@ -172,6 +175,31 @@ export const healthCheckContract = {
     )
     .output(z.void()),
 
+  /**
+   * Read the platform-wide notification policy defaults. Per-assignment
+   * rows with no override inherit these values; admin tooling reads
+   * them to populate the defaults editor. Compile-time defaults fill
+   * in any unset fields.
+   */
+  getPlatformNotificationDefaults: proc({
+    operationType: "query",
+    userType: "authenticated",
+    access: [healthCheckAccess.configuration.read],
+  }).output(NotificationPolicySchema),
+
+  /**
+   * Update the platform-wide notification policy defaults. Per-
+   * assignment rows that inherit (notificationPolicy = null) will pick
+   * up the new values on the next read.
+   */
+  setPlatformNotificationDefaults: proc({
+    operationType: "mutation",
+    userType: "authenticated",
+    access: [healthCheckAccess.configuration.manage],
+  })
+    .input(NotificationPolicySchema)
+    .output(z.void()),
+
   disassociateSystem: proc({
     operationType: "mutation",
     userType: "authenticated",
@@ -238,6 +266,8 @@ export const healthCheckContract = {
         endDate: z.date().optional(),
         /** Filter by source: "local" = core only, satellite UUID = specific satellite, undefined = all */
         sourceFilter: z.string().optional(),
+        /** Restrict runs to the listed statuses. Omitted/empty = no filter. */
+        statusFilter: z.array(HealthCheckStatusSchema).optional(),
         limit: z.number().optional().default(10),
         offset: z.number().optional().default(0),
         sortOrder: z.enum(["asc", "desc"]),
@@ -263,6 +293,8 @@ export const healthCheckContract = {
         endDate: z.date().optional(),
         /** Filter by source: "local" = core only, satellite UUID = specific satellite, undefined = all */
         sourceFilter: z.string().optional(),
+        /** Restrict runs to the listed statuses. Omitted/empty = no filter. */
+        statusFilter: z.array(HealthCheckStatusSchema).optional(),
         limit: z.number().optional().default(10),
         offset: z.number().optional().default(0),
         sortOrder: z.enum(["asc", "desc"]),

package/src/schemas.ts

@@ -207,6 +207,121 @@ export const DEFAULT_STATE_THRESHOLDS: StateThresholds = {
   unhealthy: { minFailureCount: 5 },
 };
 
+// --- Notification Policy ---
+
+/**
+ * Trigger that opens an auto-incident after a check has been
+ * continuously `unhealthy` for at least `durationMinutes`. Resets if
+ * the check recovers to non-unhealthy in between.
+ */
+export const SustainedUnhealthyTriggerSchema = z.object({
+  /** When false, this trigger is fully disabled. */
+  enabled: z.boolean().default(true),
+  /** Minimum continuous-unhealthy time before opening. */
+  durationMinutes: z.number().int().min(1).default(30),
+});
+
+export type SustainedUnhealthyTrigger = z.infer<
+  typeof SustainedUnhealthyTriggerSchema
+>;
+
+/**
+ * Trigger that opens an auto-incident when a check has transitioned
+ * to `unhealthy` at least `transitions` times within `windowMinutes`.
+ * Catches flapping that the sustained-duration trigger would miss
+ * because each unhealthy phase is too short.
+ */
+export const FlappingTriggerSchema = z.object({
+  /** When false, this trigger is fully disabled. */
+  enabled: z.boolean().default(true),
+  /** Minimum number of transitions-to-unhealthy needed in the window. */
+  transitions: z.number().int().min(1).default(3),
+  /** Sliding window in minutes the transitions are counted over. */
+  windowMinutes: z.number().int().min(1).default(60),
+});
+
+export type FlappingTrigger = z.infer<typeof FlappingTriggerSchema>;
+
+export const DEFAULT_SUSTAINED_TRIGGER: SustainedUnhealthyTrigger = {
+  enabled: true,
+  durationMinutes: 30,
+};
+
+export const DEFAULT_FLAPPING_TRIGGER: FlappingTrigger = {
+  enabled: true,
+  transitions: 3,
+  windowMinutes: 60,
+};
+
+/**
+ * Per-association notification preferences. All fields are evaluated
+ * per (system, configuration) — different checks on the same system
+ * are fully independent.
+ */
+export const NotificationPolicySchema = z.object({
+  /**
+   * When true, do not emit notifications for de-escalations (e.g.
+   * `unhealthy → degraded`). Escalations and recoveries to `healthy`
+   * still notify.
+   */
+  suppressDeEscalations: z.boolean().default(false),
+  /**
+   * When true, the configured triggers can open auto-managed incidents
+   * on the system. Setting this to false disables both triggers
+   * regardless of their individual `enabled` flags.
+   */
+  autoOpenIncidentOnUnhealthy: z.boolean().default(true),
+  /**
+   * When true, the auto-opened incident is created with
+   * `suppressNotifications` enabled so further health-state
+   * notifications for the system are silenced until the incident is
+   * resolved. Only meaningful when `autoOpenIncidentOnUnhealthy` is on.
+   */
+  useNotificationSuppression: z.boolean().default(true),
+  /**
+   * When true, no auto-incident is opened while the system has an
+   * active maintenance window with notification suppression. The
+   * system is intentionally down and shouldn't trip the on-call.
+   */
+  skipDuringMaintenance: z.boolean().default(true),
+  /**
+   * Trigger A: "this check has been unhealthy for X minutes
+   * continuously." Catches real outages.
+   */
+  sustainedUnhealthyTrigger: SustainedUnhealthyTriggerSchema.default(
+    DEFAULT_SUSTAINED_TRIGGER,
+  ),
+  /**
+   * Trigger B: "this check transitioned to unhealthy N times in M
+   * minutes." Catches persistent flapping where no individual
+   * unhealthy phase is long enough for the sustained trigger.
+   */
+  flappingTrigger: FlappingTriggerSchema.default(DEFAULT_FLAPPING_TRIGGER),
+  /**
+   * Minutes of sustained healthy state required before an auto-opened
+   * incident is auto-closed. `null` disables auto-close — the
+   * incident stays open until an operator resolves it manually.
+   */
+  autoCloseAfterMinutes: z
+    .number()
+    .int()
+    .min(1)
+    .nullable()
+    .default(30),
+});
+
+export type NotificationPolicy = z.infer<typeof NotificationPolicySchema>;
+
+export const DEFAULT_NOTIFICATION_POLICY: NotificationPolicy = {
+  suppressDeEscalations: false,
+  autoOpenIncidentOnUnhealthy: true,
+  useNotificationSuppression: true,
+  skipDuringMaintenance: true,
+  sustainedUnhealthyTrigger: DEFAULT_SUSTAINED_TRIGGER,
+  flappingTrigger: DEFAULT_FLAPPING_TRIGGER,
+  autoCloseAfterMinutes: 30,
+};
+
 export const AssociateHealthCheckSchema = z.object({
   configurationId: z.string().uuid(),
   enabled: z.boolean().default(true),
@@ -215,6 +330,8 @@ export const AssociateHealthCheckSchema = z.object({
   satelliteIds: z.array(z.string()).optional(),
   /** Whether to also run this check locally on the core instance (default: true) */
   includeLocal: z.boolean().default(true),
+  /** Per-association notification policy. Defaults applied when omitted. */
+  notificationPolicy: NotificationPolicySchema.optional(),
 });
 
 export type AssociateHealthCheck = z.infer<typeof AssociateHealthCheckSchema>;