@checkstack/healthcheck-backend 1.22.0 → 1.23.0

package/CHANGELOG.md

@@ -1,5 +1,100 @@
 # @checkstack/healthcheck-backend
 
+## 1.23.0
+
+### Minor Changes
+
+- 88f4333: Per-satellite offline threshold, connectivity notifications, and stop satellite-only checks going silent
+
+  **A satellite going offline was invisible, and so were its checks.** Three
+  related changes:
+
+  **Per-satellite offline threshold.** The 45-second global constant is now a
+  per-satellite override (**Offline after**, 2 minutes to 24 hours), because
+  tolerance is a property of the link, not of the platform: a satellite on a flaky
+  uplink needs grace that should not be forced on every other satellite. The
+  threshold is carried on every row read by `computeStatus`, so the entity read,
+  the admin list and the heartbeat monitor cannot disagree about the same
+  satellite. Additive, nullable column - existing satellites keep the default.
+
+  **Connectivity notifications.** Satellites are now a notification target with a
+  **Satellite connectivity** subscription: a warning when a satellite stops
+  heartbeating, informational when it returns. A reconnect only notifies if the
+  satellite was actually offline, so a redeploy is not an event. (The same
+  transitions remain available as `satellite.heartbeat_lost` / `.connected`
+  automation triggers for anyone wanting different routing.)
+
+  **Satellite-only checks no longer go silent.** BUG FIX: a check with
+  `includeLocal: false` whose satellites were all offline recorded NOTHING, so it
+  displayed its last known status indefinitely - a dead probe was indistinguishable
+  from a passing one. The core now records a `degraded` run with a clear message.
+  Degraded rather than unhealthy because the target may be fine; what failed is our
+  ability to observe it. Liveness that cannot be resolved is treated as "executing"
+  so a transient lookup failure cannot mark the whole fleet degraded at once.
+
+  Checks also surface staleness: a last run older than five intervals (minimum ten
+  minutes) is highlighted, so an ageing status is visible even with no run to
+  explain it. Paused checks are never stale, and neither is a RETIRED slice - one
+  whose environment was removed or whose satellite was unassigned - because
+  warning about something you retired on purpose trains operators to ignore the
+  badge.
+
+  The unobservable run does NOT notify subscribers. One offline satellite degrades
+  every check assigned to it in the same tick, and `healthy -> degraded` is an
+  escalation, so notifying per check would turn a single root cause into one alert
+  per check. The satellite's own connectivity subscription reports the cause once;
+  the runs are still recorded, so health and the UI stay honest.
+
+  Satellite liveness is cached on the shared platform cache with a 5s TTL. The
+  executor asks per tick of every satellite-only check and the read is a full
+  scan, so the uncached version scaled with the number of such checks. The TTL is
+  well below the smallest offline threshold the schema allows, so a cached answer
+  can lag a transition by one tick but never span one.
+
+  Corrects the user guide, which claimed offline satellites produced failed runs -
+  they produced nothing at all.
+
+### Patch Changes
+
+- Updated dependencies [88f4333]
+- Updated dependencies [1deaac5]
+- Updated dependencies [88f4333]
+- Updated dependencies [88f4333]
+- Updated dependencies [88f4333]
+- Updated dependencies [88f4333]
+- Updated dependencies [88f4333]
+- Updated dependencies [1deaac5]
+- Updated dependencies [56e5375]
+- Updated dependencies [88f4333]
+  - @checkstack/common@0.24.0
+  - @checkstack/healthcheck-common@1.19.1
+  - @checkstack/status-page-common@0.7.0
+  - @checkstack/incident-common@1.11.0
+  - @checkstack/maintenance-common@1.11.0
+  - @checkstack/command-backend@0.3.0
+  - @checkstack/incident-backend@1.14.0
+  - @checkstack/satellite-backend@0.10.0
+  - @checkstack/status-page-backend@0.7.0
+  - @checkstack/notification-common@1.9.0
+  - @checkstack/ai-backend@0.11.5
+  - @checkstack/backend-api@0.35.0
+  - @checkstack/satellite-common@0.12.0
+  - @checkstack/automation-backend@0.11.9
+  - @checkstack/secrets-backend@0.3.10
+  - @checkstack/ai-common@0.6.8
+  - @checkstack/cache-api@0.3.21
+  - @checkstack/catalog-backend@1.10.2
+  - @checkstack/catalog-common@2.8.2
+  - @checkstack/gitops-backend@0.5.28
+  - @checkstack/gitops-common@0.7.5
+  - @checkstack/healthcheck-execution@0.35.1
+  - @checkstack/queue-api@0.4.1
+  - @checkstack/script-packages-backend@0.4.7
+  - @checkstack/sdk@0.136.1
+  - @checkstack/secrets-common@0.3.4
+  - @checkstack/signal-common@0.3.2
+  - @checkstack/cache-utils@0.3.2
+
 ## 1.22.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/healthcheck-backend",
-  "version": "1.22.0",
+  "version": "1.23.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -14,33 +14,34 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/ai-backend": "0.11.4",
-    "@checkstack/ai-common": "0.6.7",
-    "@checkstack/automation-backend": "0.11.8",
-    "@checkstack/backend-api": "0.34.1",
-    "@checkstack/cache-api": "0.3.20",
-    "@checkstack/cache-utils": "0.3.1",
-    "@checkstack/catalog-backend": "1.10.1",
-    "@checkstack/catalog-common": "2.8.1",
-    "@checkstack/command-backend": "0.2.27",
-    "@checkstack/common": "0.23.0",
-    "@checkstack/gitops-backend": "0.5.27",
-    "@checkstack/gitops-common": "0.7.4",
-    "@checkstack/healthcheck-common": "1.19.0",
-    "@checkstack/healthcheck-execution": "0.35.0",
-    "@checkstack/incident-backend": "1.13.6",
-    "@checkstack/incident-common": "1.10.5",
-    "@checkstack/maintenance-common": "1.10.5",
-    "@checkstack/notification-common": "1.8.0",
-    "@checkstack/queue-api": "0.4.0",
-    "@checkstack/satellite-backend": "0.9.4",
-    "@checkstack/script-packages-backend": "0.4.6",
-    "@checkstack/sdk": "0.135.1",
-    "@checkstack/secrets-backend": "0.3.9",
-    "@checkstack/secrets-common": "0.3.3",
-    "@checkstack/signal-common": "0.3.1",
-    "@checkstack/status-page-backend": "0.6.6",
-    "@checkstack/status-page-common": "0.6.5",
+    "@checkstack/ai-backend": "0.11.5",
+    "@checkstack/ai-common": "0.6.8",
+    "@checkstack/automation-backend": "0.11.9",
+    "@checkstack/backend-api": "0.35.0",
+    "@checkstack/cache-api": "0.3.21",
+    "@checkstack/cache-utils": "0.3.2",
+    "@checkstack/catalog-backend": "1.10.2",
+    "@checkstack/catalog-common": "2.8.2",
+    "@checkstack/command-backend": "0.3.0",
+    "@checkstack/common": "0.24.0",
+    "@checkstack/gitops-backend": "0.5.28",
+    "@checkstack/gitops-common": "0.7.5",
+    "@checkstack/healthcheck-common": "1.19.1",
+    "@checkstack/healthcheck-execution": "0.35.1",
+    "@checkstack/incident-backend": "1.14.0",
+    "@checkstack/incident-common": "1.11.0",
+    "@checkstack/maintenance-common": "1.11.0",
+    "@checkstack/notification-common": "1.9.0",
+    "@checkstack/queue-api": "0.4.1",
+    "@checkstack/satellite-backend": "0.10.0",
+    "@checkstack/satellite-common": "0.12.0",
+    "@checkstack/script-packages-backend": "0.4.7",
+    "@checkstack/sdk": "0.136.1",
+    "@checkstack/secrets-backend": "0.3.10",
+    "@checkstack/secrets-common": "0.3.4",
+    "@checkstack/signal-common": "0.3.2",
+    "@checkstack/status-page-backend": "0.7.0",
+    "@checkstack/status-page-common": "0.7.0",
     "@hono/zod-validator": "^0.7.6",
     "@orpc/contract": "^1.14.4",
     "@orpc/server": "^1.14.4",
@@ -52,8 +53,8 @@
   },
   "devDependencies": {
     "@checkstack/drizzle-helper": "0.0.6",
-    "@checkstack/scripts": "0.7.6",
-    "@checkstack/test-utils-backend": "0.1.61",
+    "@checkstack/scripts": "0.7.7",
+    "@checkstack/test-utils-backend": "0.1.62",
     "@checkstack/tsconfig": "0.0.7",
     "@types/bun": "^1.0.0",
     "@types/pg": "^8.20.0",

package/src/index.ts

@@ -4,6 +4,7 @@ import {
   persistRunAndReact,
 } from "./queue-executor";
 import { reconcileHealthCheckJobs } from "./schedule-reconciler";
+import { SatelliteApi } from "@checkstack/satellite-common";
 import { setupRetentionJob } from "./retention-job";
 import * as schema from "./schema";
 import {
@@ -523,6 +524,15 @@ export default createBackendPlugin({
           cache,
           secretResolver,
           internalSecrets,
+          // Lets a satellite-ONLY check notice that none of its satellites are
+          // online and record a degraded run, instead of silently recording
+          // nothing and leaving its last status on screen forever. Resolved by
+          // RPC so this plugin keeps no direct dependency on satellite-backend.
+          getOnlineSatelliteIds: async () => {
+            const satelliteClient = rpcClient.forPlugin(SatelliteApi);
+            const result = await satelliteClient.getOnlineSatelliteIds();
+            return result.satelliteIds;
+          },
         });
 
         // Setup retention job for tiered storage (daily aggregation)

package/src/queue-executor.test.ts

@@ -336,6 +336,276 @@ describe("Queue-Based Health Check Executor", () => {
     });
   });
 
+  describe("executeHealthCheckJob - satellite-only checks with no online satellite", () => {
+    /**
+     * Builds a worker whose one configuration is satellite-ONLY (includeLocal
+     * false, satellites assigned) and captures the queue handler.
+     */
+    const setupSatelliteOnlyWorker = async (opts: {
+      getOnlineSatelliteIds?: () => Promise<string[]>;
+      /** Override the assignment shape to model a configuration change. */
+      configRow?: Partial<{
+        includeLocal: boolean;
+        satelliteIds: string[];
+        paused: boolean;
+      }>;
+      /** Records every subscriber notification the run would deliver. */
+      onNotify?: (input: unknown) => void;
+    }) => {
+      const mockDb = createMockDb();
+      const mockLogger = createMockLogger();
+      const mockQueueManager = createMockQueueManager();
+      const mockSignalService = createMockSignalService();
+      const mockCatalogClient = createMockCatalogClient();
+
+      // The shared mock has no `selectDistinct`; the rollup read uses it.
+      (mockDb as any).selectDistinct = mock(() => ({
+        from: mock(() => ({
+          where: mock(() => Promise.resolve([])),
+        })),
+      }));
+
+      (mockDb.select as any) = mock(() => ({
+        from: mock(() => ({
+          innerJoin: mock(() => ({
+            where: mock(() =>
+              Promise.resolve([
+                {
+                  configId: "config-1",
+                  configName: "Satellite check",
+                  strategyId: "test-strategy",
+                  config: {},
+                  collectors: [],
+                  interval: 30,
+                  enabled: true,
+                  // The shape this branch exists for: executed by satellites,
+                  // never by the core. Overridable so a test can model an
+                  // assignment change.
+                  paused: false,
+                  includeLocal: false,
+                  satelliteIds: ["sat-1", "sat-2"],
+                  environmentIds: null,
+                  ...opts.configRow,
+                },
+              ]),
+            ),
+          })),
+          where: mock(() => Promise.resolve([])),
+        })),
+      }));
+
+      const queue =
+        mockQueueManager.getQueue<HealthCheckJobPayload>("health-checks");
+      let capturedHandler:
+        | ((job: { data: HealthCheckJobPayload }) => Promise<void>)
+        | undefined;
+      (queue.consume as any) = mock(
+        async (
+          handler: (job: { data: HealthCheckJobPayload }) => Promise<void>,
+        ) => {
+          capturedHandler = handler;
+        },
+      );
+
+      await setupHealthCheckWorker({
+        db: mockDb as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["db"],
+        advisoryLock: mockAdvisoryLock,
+        registry: createMockRegistry(),
+        collectorRegistry:
+          createMockCollectorRegistry() as unknown as Parameters<
+            typeof setupHealthCheckWorker
+          >[0]["collectorRegistry"],
+        logger: mockLogger,
+        queueManager: mockQueueManager,
+        signalService: mockSignalService,
+        catalogClient: mockCatalogClient as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["catalogClient"],
+        notificationClient: {
+          notifyForSubscription: (input: unknown) => {
+            opts.onNotify?.(input);
+            return Promise.resolve({ notifiedCount: 0 });
+          },
+        } as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["notificationClient"],
+        maintenanceClient: createMockMaintenanceClient() as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["maintenanceClient"],
+        incidentClient: createMockIncidentClient() as unknown as Parameters<
+          typeof setupHealthCheckWorker
+        >[0]["incidentClient"],
+        getEmitHook: () => undefined,
+        cache: passthroughCache,
+        slowCheckRuntime: null,
+        ...(opts.getOnlineSatelliteIds
+          ? { getOnlineSatelliteIds: opts.getOnlineSatelliteIds }
+          : {}),
+      });
+
+      return { capturedHandler, mockLogger, mockSignalService };
+    };
+
+    const run = async (
+      handler:
+        | ((job: { data: HealthCheckJobPayload }) => Promise<void>)
+        | undefined,
+    ) => {
+      await handler?.({
+        data: {
+          configId: "config-1",
+          systemId: "system-1",
+          environmentId: null,
+        },
+      });
+    };
+
+    it("stays silent while at least one assigned satellite is online", async () => {
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => ["sat-2"],
+      });
+
+      await run(capturedHandler);
+
+      expect(mockLogger.debug).toHaveBeenCalledWith(
+        expect.stringContaining("satellite-only, skipping local execution"),
+      );
+      expect(mockLogger.warn).not.toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+    });
+
+    it("warns and records a run when NO assigned satellite is online", async () => {
+      // The regression this guards: the core used to return silently here, so a
+      // check whose satellites were all down kept displaying its last known
+      // status forever - a dead probe reading exactly like a passing one.
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => [],
+      });
+
+      // `persistRunAndReact` needs the full insert/aggregate/entity chain, which
+      // this file's mock database does not provide - so the call throws HERE and
+      // not in production, where it is the same function every successful run
+      // goes through. That is a mock limit, not the behaviour under test: what
+      // this pins is that the executor took the RECORD path instead of the
+      // silent one. What gets recorded is pinned separately and purely by
+      // `buildUnobservableRun` in `satellite-liveness.test.ts`.
+      await run(capturedHandler).catch(() => {});
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+      // It must NOT take the silent path.
+      expect(mockLogger.debug).not.toHaveBeenCalledWith(
+        expect.stringContaining("satellite-only, skipping local execution"),
+      );
+    });
+
+    it("stays silent when satellite liveness cannot be resolved", async () => {
+      // A transient failure to reach the satellite service must never mark every
+      // satellite-only check in the fleet degraded at once.
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => {
+          throw new Error("satellite service unreachable");
+        },
+      });
+
+      await run(capturedHandler);
+
+      expect(mockLogger.debug).toHaveBeenCalledWith(
+        expect.stringContaining("satellite-only, skipping local execution"),
+      );
+      expect(mockLogger.warn).not.toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+    });
+
+    it("stays silent when no liveness resolver is wired at all", async () => {
+      // Pre-existing behaviour for a deployment without the resolver: the core
+      // says nothing and lets the satellites report.
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({});
+
+      await run(capturedHandler);
+
+      expect(mockLogger.debug).toHaveBeenCalledWith(
+        expect.stringContaining("satellite-only, skipping local execution"),
+      );
+    });
+
+    /**
+     * An operator CHANGING an assignment must never look like a failure. These
+     * are the cases that have historically been got wrong: the platform reacts
+     * to a deliberate configuration change as though the check had broken.
+     */
+    it("records nothing when the satellites are removed from the assignment", async () => {
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        // No satellite is online at all - but the assignment no longer names
+        // any, so this check is simply not satellite-only any more.
+        getOnlineSatelliteIds: async () => [],
+        configRow: { satelliteIds: [] },
+      });
+
+      await run(capturedHandler).catch(() => {});
+
+      expect(mockLogger.warn).not.toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+    });
+
+    it("records nothing when local execution is turned back on", async () => {
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => [],
+        configRow: { includeLocal: true },
+      });
+
+      await run(capturedHandler).catch(() => {});
+
+      expect(mockLogger.warn).not.toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+    });
+
+    it("a PAUSED satellite-only check records nothing, even with every satellite offline", async () => {
+      // Paused is checked BEFORE the satellite branch, so a paused check is
+      // quiet on purpose and must not manufacture a degraded run.
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => [],
+        configRow: { paused: true },
+      });
+
+      await run(capturedHandler).catch(() => {});
+
+      expect(mockLogger.debug).toHaveBeenCalledWith(
+        expect.stringContaining("is paused, skipping execution"),
+      );
+      expect(mockLogger.warn).not.toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+    });
+
+    it("does NOT notify subscribers for the unobservable run", async () => {
+      // One offline satellite degrades EVERY check assigned to it in the same
+      // tick, and healthy -> degraded is an escalation, so notifying per check
+      // turns a single root cause into one alert per check. The satellite's own
+      // connectivity subscription reports the cause once. The RUN is still
+      // recorded - only the per-check alert is withheld.
+      const notified: unknown[] = [];
+      const { capturedHandler, mockLogger } = await setupSatelliteOnlyWorker({
+        getOnlineSatelliteIds: async () => [],
+        onNotify: (input) => notified.push(input),
+      });
+
+      await run(capturedHandler).catch(() => {});
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        expect.stringContaining("no online satellite"),
+      );
+      expect(notified).toHaveLength(0);
+    });
+  });
+
   describe("executeHealthCheckJob - collector run-context", () => {
     it("passes curated run-context to the collector (name falls back to id when configName is null)", async () => {
       const mockDb = createMockDb();

package/src/queue-executor.ts

@@ -78,6 +78,10 @@ import {
   type HealthEntityState,
 } from "./health-entity";
 import { encodeHealthEntityId } from "./health-entity-id";
+import {
+  buildUnobservableRun,
+  resolveSatelliteOnlyOutcome,
+} from "./satellite-liveness";
 import type { EntityHandle } from "@checkstack/automation-backend";
 
 type Db = SafeDatabase<typeof schema>;
@@ -628,6 +632,20 @@ export async function persistRunAndReact(params: {
   sourceLabel: string;
   /** Timestamp used for the hourly aggregate bucket (the run's execution time). */
   runTimestamp: Date;
+  /**
+   * Record the run and its transition, but do NOT notify subscribers.
+   *
+   * For a run whose cause is a single shared failure that is ALREADY notified
+   * elsewhere. The unobservable-run path is the case: one satellite going
+   * offline makes every check assigned to it degrade at once, and
+   * `healthy -> degraded` is an escalation, so without this a single satellite
+   * outage fans out into one notification per check. The satellite's own
+   * connectivity subscription names the actual root cause once.
+   *
+   * The run, the transition and the health state are still written, so the UI
+   * stays honest - only the per-check alert is withheld.
+   */
+  suppressSubscriberNotification?: boolean;
 }): Promise<void> {
   const {
     db,
@@ -655,6 +673,7 @@ export async function persistRunAndReact(params: {
     sourceId,
     sourceLabel,
     runTimestamp,
+    suppressSubscriberNotification = false,
   } = params;
 
   const envEntityId = encodeHealthEntityId({ systemId, environmentId });
@@ -762,22 +781,30 @@ export async function persistRunAndReact(params: {
       toStatus: newState.status,
     });
 
-    await notifyStateChange({
-      notificationClient,
-      systemId,
-      systemName,
-      configurationId: configId,
-      configurationName: configName,
-      previousStatus: previousStatus === "unknown" ? "healthy" : previousStatus,
-      newStatus: newState.status,
-      environmentId,
-      environmentName,
-      service,
-      catalogClient,
-      maintenanceClient,
-      incidentClient,
-      logger,
-    });
+    if (suppressSubscriberNotification) {
+      logger.debug(
+        `Recorded ${newState.status} for ${configId}/${systemId} without notifying: ` +
+          "the underlying cause is notified once at its source",
+      );
+    } else {
+      await notifyStateChange({
+        notificationClient,
+        systemId,
+        systemName,
+        configurationId: configId,
+        configurationName: configName,
+        previousStatus:
+          previousStatus === "unknown" ? "healthy" : previousStatus,
+        newStatus: newState.status,
+        environmentId,
+        environmentName,
+        service,
+        catalogClient,
+        maintenanceClient,
+        incidentClient,
+        logger,
+      });
+    }
 
     if (!isFannedOut) {
       await signalService.broadcast(SYSTEM_STATUS_CHANGED, {
@@ -859,6 +886,15 @@ async function executeHealthCheckJob(props: {
    * is skipped and the run executes exactly as before (full timeout, no lane).
    */
   slowCheckRuntime?: SlowCheckRuntime | null;
+  /**
+   * Resolves the ids of every currently-online satellite.
+   *
+   * Injected rather than imported so this module keeps no dependency on the
+   * satellite plugin, and so the unobservable-check path is testable without
+   * one. When absent, satellite-only checks behave exactly as they did before:
+   * the core stays silent and lets the satellites report.
+   */
+  getOnlineSatelliteIds?: () => Promise<string[]>;
 }): Promise<void> {
   const {
     payload,
@@ -878,6 +914,7 @@ async function executeHealthCheckJob(props: {
     secretResolver,
     internalSecrets,
     slowCheckRuntime,
+    getOnlineSatelliteIds,
   } = props;
   const { configId, systemId } = payload;
 
@@ -951,16 +988,91 @@ async function executeHealthCheckJob(props: {
       return;
     }
 
-    // If includeLocal is false and satellites are assigned, skip local execution
-    // (satellites handle execution, local core doesn't run this check)
+    // If includeLocal is false and satellites are assigned, the SATELLITES
+    // execute this check and the core does not.
+    //
+    // But "the core does not run it" is not the same as "nothing needs to
+    // happen". If every assigned satellite is offline, nobody runs it, and
+    // returning silently (as this once did) leaves the check displaying its
+    // last known status forever - a dead probe reading exactly like a passing
+    // one. So an unobservable check records a `degraded` run instead.
     if (
       !configRow.includeLocal &&
       configRow.satelliteIds &&
       configRow.satelliteIds.length > 0
     ) {
-      logger.debug(
-        `Health check ${configId} for system ${systemId} is satellite-only, skipping local execution`,
+      const satelliteIds = configRow.satelliteIds;
+      // Left UNSET (not empty) when liveness cannot be resolved: an empty list
+      // would read as "every satellite is offline" and mark the whole fleet's
+      // satellite-only checks degraded on a transient lookup failure.
+      let onlineSatelliteIds: string[] | undefined;
+      if (getOnlineSatelliteIds) {
+        try {
+          onlineSatelliteIds = await getOnlineSatelliteIds();
+        } catch (error) {
+          logger.warn(
+            `Could not resolve satellite liveness for ${configId}/${systemId}; treating as executing`,
+            error,
+          );
+        }
+      }
+
+      const outcome = resolveSatelliteOnlyOutcome({
+        satelliteIds,
+        ...(onlineSatelliteIds === undefined ? {} : { onlineSatelliteIds }),
+      });
+
+      if (outcome === "satellites-executing") {
+        logger.debug(
+          `Health check ${configId} for system ${systemId} is satellite-only, skipping local execution`,
+        );
+        return;
+      }
+
+      logger.warn(
+        `Health check ${configId} for system ${systemId} has no online satellite ` +
+          `(${satelliteIds.length} assigned); recording a degraded run so the gap is visible`,
       );
+
+      let unobservableSystemName = systemId;
+      try {
+        const system = await catalogClient.getSystem({ systemId });
+        if (system) unobservableSystemName = system.name;
+      } catch {
+        // Fall back to the id; a missing display name must not swallow the run.
+      }
+
+      await persistRunAndReact({
+        db,
+        service,
+        cache,
+        signalService,
+        notificationClient,
+        catalogClient,
+        maintenanceClient,
+        incidentClient,
+        ...(getHealthEntity ? { getHealthEntity } : {}),
+        getEmitHook,
+        collectorRegistry,
+        advisoryLock,
+        logger,
+        systemId,
+        systemName: unobservableSystemName,
+        configId,
+        ...(configRow.configName ? { configName: configRow.configName } : {}),
+        // The job payload already names the single (config, system, env) slice
+        // this tick owns, so the stale run lands on exactly the slice the
+        // satellites would have reported for.
+        ...buildUnobservableRun({
+          environmentId: payload.environmentId,
+          satelliteIds,
+        }),
+        runTimestamp: new Date(),
+        // One offline satellite degrades EVERY check assigned to it in the same
+        // tick. Notifying per check would turn a single root cause into a
+        // storm; the satellite's connectivity subscription reports it once.
+        suppressSubscriberNotification: true,
+      });
       return;
     }
 
@@ -1772,6 +1884,15 @@ export async function setupHealthCheckWorker(props: {
    * don't exercise the bulkhead), or a concrete runtime to drive it.
    */
   slowCheckRuntime?: SlowCheckRuntime | null;
+  /**
+   * Resolves the ids of every currently-online satellite.
+   *
+   * Injected rather than imported so this module keeps no dependency on the
+   * satellite plugin, and so the unobservable-check path is testable without
+   * one. When absent, satellite-only checks behave exactly as they did before:
+   * the core stays silent and lets the satellites report.
+   */
+  getOnlineSatelliteIds?: () => Promise<string[]>;
 }): Promise<void> {
   const {
     db,
@@ -1790,6 +1911,7 @@ export async function setupHealthCheckWorker(props: {
     cache,
     secretResolver,
     internalSecrets,
+    getOnlineSatelliteIds,
   } = props;
 
   // Resolve the slow-check runtime once at startup unless the caller supplied
@@ -1826,6 +1948,7 @@ export async function setupHealthCheckWorker(props: {
         secretResolver,
         internalSecrets,
         slowCheckRuntime,
+        ...(getOnlineSatelliteIds ? { getOnlineSatelliteIds } : {}),
       });
     },
     {

package/src/satellite-liveness.test.ts

@@ -0,0 +1,199 @@
+import { describe, expect, test } from "bun:test";
+import {
+  buildUnobservableResult,
+  buildUnobservableRun,
+  resolveSatelliteOnlyOutcome,
+} from "./satellite-liveness";
+
+describe("resolveSatelliteOnlyOutcome", () => {
+  test("satellites execute the check when any assigned one is online", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b"],
+        onlineSatelliteIds: ["b"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("records an unobservable run when NO assigned satellite is online", () => {
+    // The bug: this used to return silently, so the check kept displaying its
+    // last status forever and a dead probe read exactly like a passing one.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b"],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("ignores online satellites this check is not assigned to", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a"],
+        onlineSatelliteIds: ["someone-else"],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("stays silent when liveness is UNKNOWN", () => {
+    // A transient failure to reach the satellite service must never mark every
+    // satellite-only check in the fleet degraded at once. Unknown is not the
+    // same as offline, and silence is the pre-existing, safe direction.
+    expect(
+      resolveSatelliteOnlyOutcome({ satelliteIds: ["a"] }),
+    ).toBe("satellites-executing");
+  });
+
+  test("a check with no assigned satellites is treated as executing", () => {
+    // Not reachable from the caller (the branch requires assignments), but the
+    // function must not claim an empty assignment set is unobservable.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: [],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("one online satellite out of many is enough", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b", "c"],
+        onlineSatelliteIds: ["c"],
+      }),
+    ).toBe("satellites-executing");
+  });
+});
+
+describe("buildUnobservableResult", () => {
+  test("says plainly that health is UNKNOWN, not that the target is down", () => {
+    // The run is degraded because we could not observe, not because the service
+    // failed. The message must not let an operator conclude otherwise.
+    const result = buildUnobservableResult({ satelliteIds: ["a", "b"] });
+
+    expect(String(result.error)).toContain("unknown");
+    expect(String(result.error)).toContain("monitoring gap");
+    expect(String(result.error)).toContain("not a confirmed outage");
+  });
+
+  test("carries a machine-readable marker and the assignment count", () => {
+    const result = buildUnobservableResult({ satelliteIds: ["a", "b"] });
+
+    expect(result.satelliteOffline).toBe(true);
+    expect(result.assignedSatelliteCount).toBe(2);
+  });
+});
+
+describe("buildUnobservableRun", () => {
+  test("records DEGRADED, never unhealthy", () => {
+    // Unhealthy would raise incident-grade alarms about services that may be
+    // perfectly healthy, every time a satellite host reboots.
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] }).status,
+    ).toBe("degraded");
+  });
+
+  test("lands on the slice the job owns, including a concrete environment", () => {
+    // The satellites would have reported for this exact slice, so the gap has
+    // to be recorded there and not on the rollup.
+    expect(
+      buildUnobservableRun({ environmentId: "env-1", satelliteIds: ["a"] })
+        .environmentId,
+    ).toBe("env-1");
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] })
+        .environmentId,
+    ).toBeNull();
+  });
+
+  test("attributes the run to the core, not to a satellite", () => {
+    // A satellite reported nothing - the core is what noticed the gap.
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] })
+        .sourceLabel,
+    ).toBe("Local");
+  });
+
+  test("carries the explanatory result payload", () => {
+    const run = buildUnobservableRun({
+      environmentId: null,
+      satelliteIds: ["a", "b"],
+    });
+
+    expect(run.result.satelliteOffline).toBe(true);
+    expect(run.result.assignedSatelliteCount).toBe(2);
+    expect(String(run.result.error)).toContain("unknown");
+  });
+});
+
+describe("resolveSatelliteOnlyOutcome - assignment changes must not fabricate a gap", () => {
+  /**
+   * The class of bug these guard: an operator changes an assignment and the
+   * platform reacts as though something FAILED. Removing a satellite, adding
+   * local execution back, or swapping one satellite for another are all
+   * deliberate acts - none of them means "nobody is checking this".
+   */
+  test("emptying the satellite list is not an outage", () => {
+    // The executor's branch requires a non-empty list, but the predicate must
+    // agree independently - a future caller must not be able to turn a cleared
+    // assignment into a degraded run.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: [],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("swapping to a different, online satellite is not an outage", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["new-sat"],
+        onlineSatelliteIds: ["new-sat"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("one online satellite is enough even when the others were deleted", () => {
+    // A deleted satellite simply stops appearing in the online set. As long as
+    // ONE assigned satellite is still online, the check is being executed.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["deleted-sat", "live-sat"],
+        onlineSatelliteIds: ["live-sat", "unrelated-sat"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("an entirely unrelated fleet being online is NOT enough", () => {
+    // Guards the inverse mistake: "some satellite somewhere is up" must not be
+    // read as "this check is being executed".
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["mine"],
+        onlineSatelliteIds: ["someone-elses", "another"],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("an empty online set with no assignment is not an outage", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({ satelliteIds: [], onlineSatelliteIds: [] }),
+    ).toBe("satellites-executing");
+  });
+
+  test("duplicate ids in an assignment do not change the verdict", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "a"],
+        onlineSatelliteIds: ["a"],
+      }),
+    ).toBe("satellites-executing");
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "a"],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("record-unobservable");
+  });
+});

package/src/satellite-liveness.ts

@@ -0,0 +1,106 @@
+/**
+ * What a satellite-only check should do when the core reaches its tick.
+ *
+ * ## The bug this exists to close
+ *
+ * A check with `includeLocal: false` and assigned satellites is executed BY
+ * those satellites; the core's own tick has nothing to run and returned
+ * immediately. If every assigned satellite is offline, nobody executes it - and
+ * because the core recorded nothing at all, the check kept displaying whatever
+ * status it last had, indefinitely. A dead probe was indistinguishable from a
+ * passing one, which is the single worst failure mode a monitoring tool can
+ * have.
+ *
+ * So the core now records a `degraded` run instead of staying silent.
+ * `degraded` rather than `unhealthy` because the target may be perfectly
+ * healthy - what failed is our ability to observe it. Marking it unhealthy
+ * would raise incident-grade alarms about services that are fine, every time a
+ * satellite host reboots.
+ */
+export type SatelliteOnlyOutcome =
+  /** Satellites are executing this check; the core has nothing to do. */
+  | "satellites-executing"
+  /** No assigned satellite is online - record a stale run so the gap is visible. */
+  | "record-unobservable";
+
+export function resolveSatelliteOnlyOutcome({
+  satelliteIds,
+  onlineSatelliteIds,
+}: {
+  /** Satellites assigned to this check. */
+  satelliteIds: readonly string[];
+  /**
+   * Currently-online satellite ids. `undefined` when liveness could not be
+   * determined at all (no resolver wired, or the lookup failed).
+   */
+  onlineSatelliteIds?: readonly string[];
+}): SatelliteOnlyOutcome {
+  // Unknown liveness must NEVER manufacture a degraded run: a transient failure
+  // to reach the satellite service would otherwise mark every satellite-only
+  // check degraded across the fleet at once. Staying silent is the pre-existing
+  // behaviour and the safe direction for an unknown.
+  if (onlineSatelliteIds === undefined) return "satellites-executing";
+
+  // An empty assignment set has nothing to be offline. `[].some()` is false, so
+  // without this guard a check with no satellites would be reported as
+  // unobservable - a degraded run for a configuration that cannot produce one.
+  if (satelliteIds.length === 0) return "satellites-executing";
+
+  const online = new Set(onlineSatelliteIds);
+  const anyOnline = satelliteIds.some((id) => online.has(id));
+
+  return anyOnline ? "satellites-executing" : "record-unobservable";
+}
+
+/** The result payload recorded for an unobservable run. */
+export function buildUnobservableResult({
+  satelliteIds,
+}: {
+  satelliteIds: readonly string[];
+}): Record<string, unknown> {
+  const count = satelliteIds.length;
+  return {
+    error:
+      `No assigned satellite is online (${count} assigned), so this check could not be executed. ` +
+      "The target's actual health is unknown - this is a monitoring gap, not a confirmed outage.",
+    satelliteOffline: true,
+    assignedSatelliteCount: count,
+  };
+}
+
+/**
+ * The `persistRunAndReact` arguments for an unobservable run.
+ *
+ * Extracted so the RECORDED VALUES - degraded, the payload's environment slice,
+ * the local source label - are pinned by a test. The executor's mock database
+ * cannot service `persistRunAndReact`'s insert/aggregate chain, so the only way
+ * to assert what gets written is to make the decision about what to write a
+ * separate, pure step.
+ */
+export function buildUnobservableRun({
+  environmentId,
+  satelliteIds,
+}: {
+  /** The single (config, system, env) slice this job owns. */
+  environmentId: string | null;
+  satelliteIds: readonly string[];
+}): {
+  status: "degraded";
+  environmentId: string | null;
+  sourceLabel: string;
+  result: Record<string, unknown>;
+} {
+  return {
+    // Degraded, NOT unhealthy: the target may be perfectly healthy and what
+    // failed is our ability to observe it. Unhealthy would raise
+    // incident-grade alarms about healthy services on every satellite reboot.
+    status: "degraded",
+    // The job payload already names the slice the satellites would have
+    // reported for, so the gap lands exactly where the missing runs would have.
+    environmentId,
+    // Recorded by the CORE, which is what noticed the gap - not by a satellite,
+    // which by definition reported nothing.
+    sourceLabel: "Local",
+    result: buildUnobservableResult({ satelliteIds }),
+  };
+}

package/tsconfig.json

@@ -67,6 +67,9 @@
     {
       "path": "../satellite-backend"
     },
+    {
+      "path": "../satellite-common"
+    },
     {
       "path": "../script-packages-backend"
     },