@checkstack/healthcheck-backend 1.3.0 → 1.5.0

package/src/queue-executor.ts

@@ -9,6 +9,7 @@ import {
   type ConnectedClient,
   type TransportClient,
   type CollectorRunContext,
+  type AdvisoryLockService,
 } from "@checkstack/backend-api";
 import { QueueManager } from "@checkstack/queue-api";
 import {
@@ -36,6 +37,8 @@ import { IncidentApi } from "@checkstack/incident-common";
 import { NotificationApi } from "@checkstack/notification-common";
 import { healthcheckSystemSubscription } from "@checkstack/healthcheck-common";
 import { resolveRoute, type InferClient, extractErrorMessage} from "@checkstack/common";
+import { secretEnvMappingSchema } from "@checkstack/secrets-common";
+import type { SecretResolverService } from "@checkstack/secrets-backend";
 import { HealthCheckService } from "./service";
 import { healthCheckHooks } from "./hooks";
 import { incrementHourlyAggregate } from "./realtime-aggregation";
@@ -44,17 +47,13 @@ import {
   classifyTransition,
   shouldNotifyTransition,
 } from "./notification-policy";
+import { recordStateTransition } from "./state-transitions";
 import {
-  findLastAutoIncidentClose,
-  findUnhealthySince,
-  hasHealthyRunSince,
-  isMaintenanceSuppressed,
-  isTransitionToUnhealthy,
-  openAutoIncident,
-  recordUnhealthyTransition,
-  shouldOpenForFlapping,
-  shouldOpenForSustainedUnhealthy,
-} from "./auto-incident";
+  writeHealthEntity,
+  createHealthEntitySerializer,
+  type HealthEntityState,
+} from "./health-entity";
+import type { EntityHandle } from "@checkstack/automation-backend";
 
 type Db = SafeDatabase<typeof schema>;
 type CatalogClient = InferClient<typeof CatalogApi>;
@@ -62,6 +61,28 @@ type MaintenanceClient = InferClient<typeof MaintenanceApi>;
 type IncidentClient = InferClient<typeof IncidentApi>;
 type NotificationClient = InferClient<typeof NotificationApi>;
 
+/** Shape of the aggregated state returned by `getSystemHealthStatus`. */
+type AggregatedHealth = Awaited<
+  ReturnType<HealthCheckService["getSystemHealthStatus"]>
+>;
+
+/**
+ * Derive the reactive `health` entity view from the freshly-computed
+ * aggregated state. Mirrors `computeHealthEntityState` exactly: `status` is the
+ * worst-wins aggregate, `healthyChecks` counts per-check `"healthy"` statuses,
+ * and `totalChecks` is the number of enabled checks. Kept here so the
+ * `handle.mutate` write returns the SAME view the `read` accessor would have
+ * computed for the post-write state (the handle thus never re-reads).
+ */
+function toHealthEntityView(state: AggregatedHealth): HealthEntityState {
+  return {
+    status: state.status,
+    healthyChecks: state.checkStatuses.filter((c) => c.status === "healthy")
+      .length,
+    totalChecks: state.checkStatuses.length,
+  };
+}
+
 /**
  * Emit the checkCompleted hook if available, plus the narrower
  * `checkFailed` hook when the result wasn't `healthy` (so operators
@@ -171,222 +192,12 @@ export async function scheduleHealthCheck(props: {
   });
 }
 
-/**
- * After every check run, evaluate the per-check auto-incident
- * triggers. Either trigger can independently open an incident:
- *
- * - **flapping**: this just-completed run was a transition to
- *   unhealthy AND `N` such transitions have happened within the
- *   configured window.
- * - **sustained**: the check is currently unhealthy AND has been so
- *   continuously for at least the configured duration.
- *
- * Both triggers honour the require-recovery rule: after the most
- * recent auto-incident close (manual or auto), no new auto-incident
- * opens until the check has logged at least one healthy run. This
- * stops a manual close → still-unhealthy → re-open loop.
- *
- * Active maintenance with suppression skips both triggers when the
- * policy opts in.
- */
-async function maybeOpenAutoIncidentForCheck(props: {
-  db: Db;
-  service: HealthCheckService;
-  incidentClient: IncidentClient;
-  maintenanceClient: MaintenanceClient;
-  logger: Logger;
-  systemId: string;
-  systemName: string;
-  configurationId: string;
-  configurationName: string;
-  /**
-   * Same closure-based getter the queue executor uses elsewhere; let
-   * us fire the `flapping_detected` automation hook from inside the
-   * flapping evaluator without re-threading `emitHook` through every
-   * intermediate caller. Optional — when absent, the hook simply
-   * doesn't fire (e.g. in unit tests that don't care about it).
-   */
-  getEmitHook?: () => EmitHookFn | undefined;
-  previousState: {
-    checkStatuses: Array<{
-      configurationId: string;
-      status: HealthCheckStatus;
-    }>;
-  };
-  newState: {
-    checkStatuses: Array<{
-      configurationId: string;
-      status: HealthCheckStatus;
-    }>;
-  };
-}): Promise<void> {
-  const {
-    db,
-    service,
-    incidentClient,
-    maintenanceClient,
-    logger,
-    systemId,
-    systemName,
-    configurationId,
-    configurationName,
-    getEmitHook,
-    previousState,
-    newState,
-  } = props;
-
-  const next = newState.checkStatuses.find(
-    (c) => c.configurationId === configurationId,
-  );
-  // Only auto-incident logic applies when the check is currently
-  // unhealthy — both triggers require it.
-  if (!next || next.status !== "unhealthy") return;
-
-  const prev = previousState.checkStatuses.find(
-    (c) => c.configurationId === configurationId,
-  );
-  const isTransition = isTransitionToUnhealthy(prev?.status, next.status);
-
-  let policy;
-  try {
-    policy = await service.getAssignmentNotificationPolicy({
-      systemId,
-      configurationId,
-    });
-  } catch (error) {
-    logger.warn(
-      `Failed to load policy for auto-incident decision (${systemId}/${configurationId}):`,
-      error,
-    );
-    return;
-  }
-
-  if (!policy.autoOpenIncidentOnUnhealthy) return;
-
-  // Honour active maintenance windows — operators have explicitly
-  // said the system is down on purpose.
-  if (policy.skipDuringMaintenance) {
-    const suppressed = await isMaintenanceSuppressed({
-      maintenanceClient,
-      systemId,
-      logger,
-    });
-    if (suppressed) {
-      logger.debug(
-        `Skipping auto-incident for ${systemId}/${configurationId}: active maintenance`,
-      );
-      return;
-    }
-  }
-
-  // Require-recovery: if there's a prior closed auto-incident for
-  // this assignment, the check must have logged at least one healthy
-  // run since the close before we can open another one. Without this,
-  // an operator's manual close on a still-broken system would loop.
-  const lastCloseAt = await findLastAutoIncidentClose({
-    db,
-    systemId,
-    configurationId,
-  });
-  if (lastCloseAt) {
-    const recovered = await hasHealthyRunSince({
-      db,
-      systemId,
-      configurationId,
-      since: lastCloseAt,
-    });
-    if (!recovered) {
-      return;
-    }
-  }
-
-  // Record the transition (if any) and evaluate the flapping trigger
-  // against transitions that happened after the last close window.
-  let flappingOpens = false;
-  if (isTransition) {
-    try {
-      const count = await recordUnhealthyTransition({
-        db,
-        configurationId,
-        systemId,
-        windowMinutes: policy.flappingTrigger.windowMinutes,
-        since: lastCloseAt,
-      });
-      flappingOpens = shouldOpenForFlapping({
-        policy,
-        recentTransitionCount: count,
-      });
-
-      // Fire the informational `flapping_detected` automation hook
-      // independently of the auto-incident decision: an operator may
-      // care about flapping even with the auto-incident pipeline
-      // turned off.
-      if (
-        policy.flappingTrigger.enabled &&
-        count >= policy.flappingTrigger.transitions
-      ) {
-        const emit = getEmitHook?.();
-        if (emit) {
-          try {
-            await emit(healthCheckHooks.flappingDetected, {
-              systemId,
-              configurationId,
-              transitionCount: count,
-              windowMinutes: policy.flappingTrigger.windowMinutes,
-              timestamp: new Date().toISOString(),
-            });
-          } catch (error) {
-            logger.warn(
-              `Failed to emit healthcheck.flapping_detected hook for ${systemId}/${configurationId}:`,
-              error,
-            );
-          }
-        }
-      }
-    } catch (error) {
-      logger.warn(
-        `Failed to record unhealthy transition for ${systemId}/${configurationId}:`,
-        error,
-      );
-    }
-  }
-
-  // Evaluate the sustained-duration trigger on every run while the
-  // check is unhealthy (not just on transition).
-  let sustainedOpens = false;
-  if (policy.sustainedUnhealthyTrigger.enabled) {
-    const unhealthySince = await findUnhealthySince({
-      db,
-      configurationId,
-      systemId,
-      since: lastCloseAt,
-    });
-    if (unhealthySince) {
-      sustainedOpens = shouldOpenForSustainedUnhealthy({
-        policy,
-        unhealthyForMs: Date.now() - unhealthySince.getTime(),
-      });
-    }
-  }
-
-  if (!flappingOpens && !sustainedOpens) return;
-
-  const reason = flappingOpens
-    ? `flapping: ≥${policy.flappingTrigger.transitions} transitions in ${policy.flappingTrigger.windowMinutes} min`
-    : `unhealthy ≥${policy.sustainedUnhealthyTrigger.durationMinutes} min continuously`;
-
-  await openAutoIncident({
-    db,
-    incidentClient,
-    logger,
-    systemId,
-    systemName,
-    configurationId,
-    configurationName,
-    policy,
-    reason,
-  });
-}
+// Flapping detection no longer lives here. It moved into the automation
+// engine as a windowed-count gate on the `healthcheck.system_health_changed`
+// trigger (raw aggregated-health change + `filter` +
+// `window: { count, minutes, refire: "once" }`). The queue executor emits only
+// the raw per-system health change (via the reactive `health` entity deriver,
+// unchanged); the engine does the counting.
 
 /**
  * Notify system subscribers about a health state change.
@@ -565,6 +376,7 @@ async function notifyStateChange(props: {
 async function executeHealthCheckJob(props: {
   payload: HealthCheckJobPayload;
   db: Db;
+  advisoryLock: AdvisoryLockService;
   registry: HealthCheckRegistry;
   collectorRegistry: CollectorRegistry;
   logger: Logger;
@@ -575,10 +387,26 @@ async function executeHealthCheckJob(props: {
   incidentClient: IncidentClient;
   getEmitHook: () => EmitHookFn | undefined;
   cache: HealthCheckCache;
+  /**
+   * Resolver for the reactive `health` entity handle (§10.3). Returns the
+   * handle once automation-backend has bound the entity store; `undefined`
+   * during version skew / tests. Mirrors the `getEmitHook` closure pattern.
+   * The entity is PLUGIN-BACKED + COMPUTED — there is no keyed store; the
+   * durable run/aggregate write IS the entity write (see `writeHealthEntity`).
+   */
+  getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
+  /**
+   * Central secret resolver. When set, a collector declaring a `secretEnv`
+   * has it resolved + injected for this centrally-executed run; the
+   * collector masks the values out of its output. Optional for version-skew
+   * / test isolation.
+   */
+  secretResolver?: SecretResolverService;
 }): Promise<void> {
   const {
     payload,
     db,
+    advisoryLock,
     registry,
     collectorRegistry,
     logger,
@@ -588,13 +416,25 @@ async function executeHealthCheckJob(props: {
     maintenanceClient,
     incidentClient,
     getEmitHook,
+    getHealthEntity,
     cache,
+    secretResolver,
   } = props;
   const { configId, systemId } = payload;
 
   // Create service for aggregated state evaluation
   const service = new HealthCheckService(db, registry, collectorRegistry);
 
+  // Per-system serializer for the reactive health mutate (§10.3): a
+  // transaction-scoped advisory lock keyed `health:<systemId>` wraps the
+  // snapshot-prev + apply + diff + emit so concurrent evaluations of one
+  // system (multiple per-config jobs across pods, or at-least-once
+  // redelivery) can't double-emit a single logical transition. Bound to this
+  // job's systemId below at every `writeHealthEntity` call.
+  const serializeHealthWrite = createHealthEntitySerializer({ advisoryLock })(
+    systemId,
+  );
+
   // Capture aggregated state BEFORE this run for comparison
   const previousState = await service.getSystemHealthStatus(systemId);
   const previousStatus = previousState.status;
@@ -725,11 +565,31 @@ async function executeHealthCheckJob(props: {
             const storageKey = collectorEntry.id;
 
             try {
+              // Resolve the collector's declared secretEnv for THIS run
+              // (central execution). The collector injects it and masks the
+              // values out of its output. A missing required secret throws
+              // and fails the collector clearly.
+              let secretEnv: Record<string, string> | undefined;
+              const declared = secretEnvMappingSchema.safeParse(
+                (collectorEntry.config as { secretEnv?: unknown }).secretEnv,
+              );
+              if (
+                secretResolver &&
+                declared.success &&
+                Object.keys(declared.data).length > 0
+              ) {
+                const resolved = await secretResolver.resolveForRun({
+                  secretEnv: declared.data,
+                });
+                secretEnv = resolved.env;
+              }
+
               const collectorResult = await registered.collector.execute({
                 config: collectorEntry.config,
                 client: connectedClient!.client,
                 pluginId: configRow.strategyId,
                 runContext,
+                ...(secretEnv ? { secretEnv } : {}),
               });
 
               // Check for collector-level error
@@ -860,26 +720,44 @@ async function executeHealthCheckJob(props: {
         },
       };
 
-      await db.insert(healthCheckRuns).values({
-        configurationId: configId,
+      // Persist the run + aggregate THROUGH the reactive `health` entity:
+      // `apply` does the durable write and returns the freshly-computed view.
+      // The framework snapshots `prev` via `read` BEFORE this insert, so a real
+      // status change emits exactly one correct `ENTITY_CHANGED` (§10.3). The
+      // computed aggregated state is stashed for the transition/notify path.
+      let newState!: AggregatedHealth;
+      await writeHealthEntity({
+        handle: getHealthEntity?.(),
         systemId,
-        status: result.status,
-        latencyMs: result.latencyMs,
-        result: { ...result } as Record<string, unknown>,
-        sourceId: undefined,
-        sourceLabel: "Local",
-      });
+        apply: async () => {
+          await db.insert(healthCheckRuns).values({
+            configurationId: configId,
+            systemId,
+            status: result.status,
+            latencyMs: result.latencyMs,
+            result: { ...result } as Record<string, unknown>,
+            sourceId: undefined,
+            sourceLabel: "Local",
+          });
 
-      await incrementHourlyAggregate({
-        db,
-        systemId,
-        configurationId: configId,
-        status: result.status,
-        latencyMs: result.latencyMs,
-        runTimestamp: new Date(),
-        result: { ...result } as Record<string, unknown>,
-        collectorRegistry,
-        sourceLabel: "Local",
+          await incrementHourlyAggregate({
+            db,
+            systemId,
+            configurationId: configId,
+            status: result.status,
+            latencyMs: result.latencyMs,
+            runTimestamp: new Date(),
+            result: { ...result } as Record<string, unknown>,
+            collectorRegistry,
+            sourceLabel: "Local",
+          });
+
+          newState = await service.getSystemHealthStatus(systemId);
+          return toHealthEntityView(newState);
+        },
+        serialize: serializeHealthWrite,
+        onError: (error) =>
+          logger.warn(`Failed to mirror health entity for ${systemId}`, error),
       });
 
       logger.debug(
@@ -899,8 +777,17 @@ async function executeHealthCheckJob(props: {
         latencyMs: result.latencyMs,
       });
 
-      const newState = await service.getSystemHealthStatus(systemId);
       if (newState.status !== previousStatus) {
+        // Record the aggregate transition so the sensing layer has a
+        // reliable "in status since" for every status (Wave 2).
+        await recordStateTransition({
+          db,
+          systemId,
+          configurationId: configId,
+          fromStatus: previousStatus,
+          toStatus: newState.status,
+        });
+
         await notifyStateChange({
           notificationClient,
           systemId,
@@ -916,24 +803,6 @@ async function executeHealthCheckJob(props: {
         });
       }
 
-      // Per-check auto-incident: runs whether or not the aggregate
-      // changed (a check can transition to unhealthy without flipping
-      // the aggregate if another check is already unhealthy).
-      await maybeOpenAutoIncidentForCheck({
-        db,
-        service,
-        incidentClient,
-        maintenanceClient,
-        logger,
-        systemId,
-        systemName,
-        configurationId: configId,
-        configurationName: configRow.configName,
-        getEmitHook,
-        previousState,
-        newState,
-      });
-
       return;
     } finally {
       if (connectedClient) {
@@ -962,28 +831,48 @@ async function executeHealthCheckJob(props: {
       },
     };
 
-    // Store result (spread to convert structured type to plain record for jsonb)
-    await db.insert(healthCheckRuns).values({
-      configurationId: configId,
+    // Persist the run + aggregate THROUGH the reactive `health` entity on
+    // every run (§10.3): `apply` does the durable write (insert + hourly
+    // aggregate) and returns the freshly-computed view. The framework
+    // snapshots `prev` via the COMPUTE-ON-READ accessor BEFORE this insert, so
+    // an unchanged aggregate is a no-op and a real status change drives the
+    // directional/umbrella trigger events via `deriveHealthTriggerEvents` —
+    // exactly one correct `ENTITY_CHANGED` with accurate prev → next.
+    let newState!: AggregatedHealth;
+    await writeHealthEntity({
+      handle: getHealthEntity?.(),
       systemId,
-      status: result.status,
-      latencyMs: result.latencyMs,
-      result: { ...result } as Record<string, unknown>,
-      sourceId: undefined,
-      sourceLabel: "Local",
-    });
+      apply: async () => {
+        // Store result (spread to convert structured type to plain record for jsonb)
+        await db.insert(healthCheckRuns).values({
+          configurationId: configId,
+          systemId,
+          status: result.status,
+          latencyMs: result.latencyMs,
+          result: { ...result } as Record<string, unknown>,
+          sourceId: undefined,
+          sourceLabel: "Local",
+        });
 
-    // Trigger incremental hourly aggregation
-    await incrementHourlyAggregate({
-      db,
-      systemId,
-      configurationId: configId,
-      status: result.status,
-      latencyMs: result.latencyMs,
-      runTimestamp: new Date(),
-      result: { ...result } as Record<string, unknown>,
-      collectorRegistry,
-      sourceLabel: "Local",
+        // Trigger incremental hourly aggregation
+        await incrementHourlyAggregate({
+          db,
+          systemId,
+          configurationId: configId,
+          status: result.status,
+          latencyMs: result.latencyMs,
+          runTimestamp: new Date(),
+          result: { ...result } as Record<string, unknown>,
+          collectorRegistry,
+          sourceLabel: "Local",
+        });
+
+        newState = await service.getSystemHealthStatus(systemId);
+        return toHealthEntityView(newState);
+      },
+      serialize: serializeHealthWrite,
+      onError: (error) =>
+        logger.warn(`Failed to mirror health entity for ${systemId}`, error),
     });
 
     logger.debug(
@@ -1013,9 +902,17 @@ async function executeHealthCheckJob(props: {
       result: (result.metadata?.collectors as Record<string, unknown>) ?? undefined,
     });
 
-    // Check if aggregated state changed and notify subscribers
-    const newState = await service.getSystemHealthStatus(systemId);
     if (newState.status !== previousStatus) {
+      // Record the aggregate transition so the sensing layer has a
+      // reliable "in status since" for every status (Wave 2).
+      await recordStateTransition({
+        db,
+        systemId,
+        configurationId: configId,
+        fromStatus: previousStatus,
+        toStatus: newState.status,
+      });
+
       await notifyStateChange({
         notificationClient,
         systemId,
@@ -1037,77 +934,13 @@ async function executeHealthCheckJob(props: {
         newStatus: newState.status,
       });
 
-      // Emit integration hooks for external integrations
-      const emitHook = getEmitHook();
-      if (emitHook) {
-        const healthyChecks = newState.checkStatuses.filter(
-          (c) => c.status === "healthy",
-        ).length;
-        const totalChecks = newState.checkStatuses.length;
-        const timestamp = new Date().toISOString();
-
-        if (newState.status === "healthy" && previousStatus !== "healthy") {
-          // Recovery: system became healthy
-          await emitHook(healthCheckHooks.systemHealthy, {
-            systemId,
-            previousStatus,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-          logger.debug(
-            `Emitted systemHealthy hook: ${previousStatus} → ${newState.status}`,
-          );
-        } else if (
-          previousStatus === "healthy" &&
-          newState.status !== "healthy"
-        ) {
-          // Degradation: system went from healthy to unhealthy/degraded
-          await emitHook(healthCheckHooks.systemDegraded, {
-            systemId,
-            previousStatus,
-            newStatus: newState.status,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-          logger.debug(
-            `Emitted systemDegraded hook: ${previousStatus} → ${newState.status}`,
-          );
-        }
-
-        // Umbrella hook — fires on every transition. Emitted alongside
-        // the directional hooks so existing subscribers stay unchanged
-        // while new automation triggers can react to any change.
-        if (previousStatus !== newState.status) {
-          await emitHook(healthCheckHooks.systemHealthChanged, {
-            systemId,
-            previousStatus,
-            newStatus: newState.status,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-        }
-      }
+      // The directional + umbrella system-health hooks were removed in
+      // Phase 4 (§10.3): the `health` entity mirror above is the single
+      // source of truth, and its change deriver fires the
+      // `healthcheck.system_degraded` / `_healthy` / `_health_changed`
+      // trigger events through Stage-1 routing. Nothing to emit here.
     }
 
-    // Per-check auto-incident: see comment on the failed-execution path.
-    await maybeOpenAutoIncidentForCheck({
-      db,
-      service,
-      incidentClient,
-      maintenanceClient,
-      logger,
-      systemId,
-      systemName,
-      configurationId: configId,
-      configurationName: configRow.configName,
-      getEmitHook,
-      previousState,
-      newState,
-    });
-
     // Note: No manual rescheduling needed - recurring job handles it automatically
   } catch (error) {
     logger.error(
@@ -1115,27 +948,48 @@ async function executeHealthCheckJob(props: {
       error,
     );
 
-    // Store failure (no latencyMs for failures)
-    await db.insert(healthCheckRuns).values({
-      configurationId: configId,
+    // Persist the failure run + aggregate THROUGH the reactive `health`
+    // entity: `apply` does the durable write and returns the freshly-computed
+    // view. The framework snapshots `prev` via the compute-on-read accessor
+    // BEFORE this insert, so a real status change emits exactly one correct
+    // `ENTITY_CHANGED` (§10.3). See the success path for the full rationale.
+    let newState!: AggregatedHealth;
+    await writeHealthEntity({
+      handle: getHealthEntity?.(),
       systemId,
-      status: "unhealthy",
-      result: { error: String(error) } as Record<string, unknown>,
-      sourceId: undefined,
-      sourceLabel: "Local",
-    });
+      apply: async () => {
+        // Store failure (no latencyMs for failures)
+        await db.insert(healthCheckRuns).values({
+          configurationId: configId,
+          systemId,
+          status: "unhealthy",
+          result: { error: String(error) } as Record<string, unknown>,
+          sourceId: undefined,
+          sourceLabel: "Local",
+        });
 
-    // Trigger incremental hourly aggregation
-    await incrementHourlyAggregate({
-      db,
-      systemId,
-      configurationId: configId,
-      status: "unhealthy",
-      latencyMs: undefined,
-      runTimestamp: new Date(),
-      // No collector data for error cases
-      collectorRegistry,
-      sourceLabel: "Local",
+        // Trigger incremental hourly aggregation
+        await incrementHourlyAggregate({
+          db,
+          systemId,
+          configurationId: configId,
+          status: "unhealthy",
+          latencyMs: undefined,
+          runTimestamp: new Date(),
+          // No collector data for error cases
+          collectorRegistry,
+          sourceLabel: "Local",
+        });
+
+        newState = await service.getSystemHealthStatus(systemId);
+        return toHealthEntityView(newState);
+      },
+      serialize: serializeHealthWrite,
+      onError: (mirrorError) =>
+        logger.warn(
+          `Failed to mirror health entity for ${systemId}`,
+          mirrorError,
+        ),
     });
 
     // Try to fetch names for the enriched signal (best-effort)
@@ -1179,9 +1033,17 @@ async function executeHealthCheckJob(props: {
       result: undefined,
     });
 
-    // Check if aggregated state changed and notify subscribers
-    const newState = await service.getSystemHealthStatus(systemId);
     if (newState.status !== previousStatus) {
+      // Record the aggregate transition so the sensing layer has a
+      // reliable "in status since" for every status (Wave 2).
+      await recordStateTransition({
+        db,
+        systemId,
+        configurationId: configId,
+        fromStatus: previousStatus,
+        toStatus: newState.status,
+      });
+
       await notifyStateChange({
         notificationClient,
         systemId,
@@ -1203,83 +1065,20 @@ async function executeHealthCheckJob(props: {
         newStatus: newState.status,
       });
 
-      // Emit integration hooks for external integrations
-      const emitHook = getEmitHook();
-      if (emitHook) {
-        const healthyChecks = newState.checkStatuses.filter(
-          (c) => c.status === "healthy",
-        ).length;
-        const totalChecks = newState.checkStatuses.length;
-        const timestamp = new Date().toISOString();
-
-        if (newState.status === "healthy" && previousStatus !== "healthy") {
-          // Recovery: system became healthy
-          await emitHook(healthCheckHooks.systemHealthy, {
-            systemId,
-            previousStatus,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-          logger.debug(
-            `Emitted systemHealthy hook: ${previousStatus} → ${newState.status}`,
-          );
-        } else if (
-          previousStatus === "healthy" &&
-          newState.status !== "healthy"
-        ) {
-          // Degradation: system went from healthy to unhealthy/degraded
-          await emitHook(healthCheckHooks.systemDegraded, {
-            systemId,
-            previousStatus,
-            newStatus: newState.status,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-          logger.debug(
-            `Emitted systemDegraded hook: ${previousStatus} → ${newState.status}`,
-          );
-        }
-
-        // Umbrella hook — fires on every transition. Emitted alongside
-        // the directional hooks so existing subscribers stay unchanged
-        // while new automation triggers can react to any change.
-        if (previousStatus !== newState.status) {
-          await emitHook(healthCheckHooks.systemHealthChanged, {
-            systemId,
-            previousStatus,
-            newStatus: newState.status,
-            healthyChecks,
-            totalChecks,
-            timestamp,
-          });
-        }
-      }
+      // The directional + umbrella system-health hooks were removed in
+      // Phase 4 (§10.3): the `health` entity mirror above is the single
+      // source of truth, and its change deriver fires the
+      // `healthcheck.system_degraded` / `_healthy` / `_health_changed`
+      // trigger events through Stage-1 routing. Nothing to emit here.
     }
 
-    // Per-check auto-incident: see comment on the failed-execution path.
-    await maybeOpenAutoIncidentForCheck({
-      db,
-      service,
-      incidentClient,
-      maintenanceClient,
-      logger,
-      systemId,
-      systemName,
-      configurationId: configId,
-      configurationName: configName,
-      getEmitHook,
-      previousState,
-      newState,
-    });
-
     // Note: No manual rescheduling needed - recurring job handles it automatically
   }
 }
 
 export async function setupHealthCheckWorker(props: {
   db: Db;
+  advisoryLock: AdvisoryLockService;
   registry: HealthCheckRegistry;
   collectorRegistry: CollectorRegistry;
   logger: Logger;
@@ -1290,10 +1089,13 @@ export async function setupHealthCheckWorker(props: {
   maintenanceClient: MaintenanceClient;
   incidentClient: IncidentClient;
   getEmitHook: () => EmitHookFn | undefined;
+  getHealthEntity?: () => EntityHandle<HealthEntityState> | undefined;
   cache: HealthCheckCache;
+  secretResolver?: SecretResolverService;
 }): Promise<void> {
   const {
     db,
+    advisoryLock,
     registry,
     collectorRegistry,
     logger,
@@ -1304,7 +1106,9 @@ export async function setupHealthCheckWorker(props: {
     maintenanceClient,
     incidentClient,
     getEmitHook,
+    getHealthEntity,
     cache,
+    secretResolver,
   } = props;
 
   const queue =
@@ -1316,6 +1120,7 @@ export async function setupHealthCheckWorker(props: {
       await executeHealthCheckJob({
         payload: job.data,
         db,
+        advisoryLock,
         registry,
         collectorRegistry,
         logger,
@@ -1325,7 +1130,9 @@ export async function setupHealthCheckWorker(props: {
         maintenanceClient,
         incidentClient,
         getEmitHook,
+        getHealthEntity,
         cache,
+        secretResolver,
       });
     },
     {