@checkstack/healthcheck-backend 1.21.3 → 1.23.0

package/src/router-satellite-ingest.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect, mock } from "bun:test";
+import { call } from "@orpc/server";
+import { createMockRpcContext } from "@checkstack/backend-api";
+import { createHealthCheckRouter } from "./router";
+import { createStubHealthCheckCache } from "./cache-test-stub";
+import type { HealthRunReaction } from "./queue-executor";
+
+/**
+ * A satellite result MUST drive the SAME core post-run path a local run does:
+ * the reactive `health` entity write, the notification, the automation hooks,
+ * etc. The router routes it there via the injected `reactToSatelliteRun`
+ * reactor. These tests pin that wiring so ingest can never silently regress to
+ * an insert-only path (which is what left satellite-detected outages silent).
+ */
+
+// A service principal - ingestSatelliteResult is a backend-to-backend proc.
+const serviceContext = () =>
+  createMockRpcContext({
+    user: { type: "service", pluginId: "satellite-backend" } as never,
+  });
+
+function buildRouter(opts: {
+  reactToSatelliteRun?: (run: HealthRunReaction) => Promise<void>;
+  transaction?: ReturnType<typeof mock>;
+}) {
+  // db.select(...).from(...).where(...) resolves the config row (name +
+  // collectors) that processSatelliteResult reads.
+  const database = {
+    select: mock(() => ({
+      from: mock(() => ({
+        where: mock(() =>
+          Promise.resolve([{ name: "Ping", collectors: [] }]),
+        ),
+      })),
+    })),
+    transaction:
+      opts.transaction ??
+      mock(async () => {
+        /* not expected in the reactor path */
+      }),
+  };
+
+  const collectorRegistry = {
+    register: mock(() => {}),
+    getCollector: mock(() => undefined),
+    getCollectors: mock(() => []),
+  };
+
+  const catalogClient = {
+    getSystem: mock(async () => ({ id: "sys-1", name: "API Server" })),
+  };
+
+  return createHealthCheckRouter({
+    database: database as never,
+    registry: { getStrategy: mock(() => undefined) } as never,
+    collectorRegistry: collectorRegistry as never,
+    gitOpsClient: {} as never,
+    getEmitHook: () => undefined,
+    cache: createStubHealthCheckCache(),
+    configService: {} as never,
+    catalogClient: catalogClient as never,
+    maintenanceClient: {} as never,
+    logger: {
+      debug: mock(() => {}),
+      info: mock(() => {}),
+      warn: mock(() => {}),
+      error: mock(() => {}),
+    } as never,
+    ...(opts.reactToSatelliteRun
+      ? { reactToSatelliteRun: opts.reactToSatelliteRun }
+      : {}),
+  });
+}
+
+const ingestInput = {
+  configId: "config-1",
+  systemId: "sys-1",
+  status: "unhealthy" as const,
+  latencyMs: 123,
+  result: {
+    status: "unhealthy" as const,
+    latencyMs: 123,
+    message: "Check failed",
+    metadata: { connected: true, collectors: {} },
+  },
+  executedAt: "2026-07-03T10:00:00.000Z",
+  sourceId: "sat-eu",
+  sourceLabel: "EU West",
+  environmentId: "env-prod",
+};
+
+describe("ingestSatelliteResult - drives the shared post-run path", () => {
+  it("routes a satellite result through reactToSatelliteRun with its processed payload", async () => {
+    const runs: HealthRunReaction[] = [];
+    const router = buildRouter({
+      reactToSatelliteRun: async (run) => {
+        runs.push(run);
+      },
+    });
+
+    await call(router.ingestSatelliteResult, ingestInput, {
+      context: serviceContext(),
+    });
+
+    expect(runs).toHaveLength(1);
+    const run = runs[0];
+    // The run carries the satellite's SOURCE + environment, and its display
+    // name was resolved for the notification the shared path will send.
+    expect(run.systemId).toBe("sys-1");
+    expect(run.systemName).toBe("API Server");
+    expect(run.configId).toBe("config-1");
+    expect(run.configName).toBe("Ping");
+    expect(run.environmentId).toBe("env-prod");
+    expect(run.status).toBe("unhealthy");
+    expect(run.sourceId).toBe("sat-eu");
+    expect(run.sourceLabel).toBe("EU West");
+    expect(run.runTimestamp).toEqual(new Date("2026-07-03T10:00:00.000Z"));
+  });
+
+  it("falls back to an insert-only persist when no reactor is wired", async () => {
+    // Record that the insert-only fallback opened a transaction, without
+    // executing its body (the aggregate internals need a full db mock and are
+    // covered elsewhere); the point is that ingest still records the run.
+    const transaction = mock(async () => {});
+    const router = buildRouter({ transaction });
+
+    await call(router.ingestSatelliteResult, ingestInput, {
+      context: serviceContext(),
+    });
+
+    // The insert-only fallback ran (a run is still recorded), but there is no
+    // reactor path to drive notifications/automations - which is exactly why
+    // the real host always wires `reactToSatelliteRun`.
+    expect(transaction).toHaveBeenCalledTimes(1);
+  });
+});

package/src/router.ts

@@ -46,6 +46,7 @@ import { CatalogApi } from "@checkstack/catalog-common";
 import { MaintenanceApi } from "@checkstack/maintenance-common";
 import type { Logger } from "@checkstack/backend-api";
 import type { HealthCheckCache } from "./cache";
+import type { HealthRunReaction } from "./queue-executor";
 import {
   applySystemHealthOverrides,
   type SystemHealthOverrideReader,
@@ -104,6 +105,17 @@ export const createHealthCheckRouter = (opts: {
    * Optional so tests / no-incident deployments simply skip the fold.
    */
   incidentHealthOverrideReader?: SystemHealthOverrideReader;
+  /**
+   * Drives the SHARED core post-run path for an ingested satellite result -
+   * the reactive `health` entity write, cache reconcile, realtime signal,
+   * automation hooks, transition record, and subscriber notification - so a
+   * satellite-detected change reacts exactly like a local run. The host binds
+   * the service dependencies once (via a `persistRunAndReact` closure) and
+   * passes this narrowed reactor, so the local and satellite callers cannot
+   * pass different deps and drift. Optional so tests may omit it, in which case
+   * ingest falls back to an insert-only persist.
+   */
+  reactToSatelliteRun?: (run: HealthRunReaction) => Promise<void>;
 }) => {
   const {
     database,
@@ -118,6 +130,7 @@ export const createHealthCheckRouter = (opts: {
     signalService,
     recomputeSystemRollupHealth,
     incidentHealthOverrideReader,
+    reactToSatelliteRun,
   } = opts;
   // Create service instance once - shared across all handlers
   const service = new HealthCheckService(
@@ -612,6 +625,7 @@ export const createHealthCheckRouter = (opts: {
         enabled: input.body.enabled,
         stateThresholds: input.body.stateThresholds,
         satelliteIds: input.body.satelliteIds,
+        satelliteEnvironmentIds: input.body.satelliteEnvironmentIds,
         environmentIds: input.body.environmentIds,
         includeLocal: input.body.includeLocal,
         notificationPolicy: input.body.notificationPolicy,
@@ -634,6 +648,7 @@ export const createHealthCheckRouter = (opts: {
         enabled: input.enabled,
         stateThresholds: input.stateThresholds,
         satelliteIds: input.satelliteIds,
+        satelliteEnvironmentIds: input.satelliteEnvironmentIds,
         environmentIds: input.environmentIds,
         includeLocal: input.includeLocal,
         notificationPolicy: input.notificationPolicy,
@@ -899,10 +914,56 @@ export const createHealthCheckRouter = (opts: {
 
     ingestSatelliteResult: os.ingestSatelliteResult.handler(
       async ({ input }) => {
-        await service.ingestSatelliteResult(input);
-        // A satellite result writes a new run for this system, so the
-        // cached aggregate status is now stale.
-        await cache.invalidateSystem(input.systemId);
+        // Process the satellite's raw result (evaluate assertions, strip
+        // ephemeral fields, resolve the check name) WITHOUT persisting...
+        const { status, resultRecord, configName } =
+          await service.processSatelliteResult({
+            configId: input.configId,
+            status: input.status,
+            result: input.result,
+          });
+
+        if (reactToSatelliteRun) {
+          // ...then persist + REACT through the SAME core post-run path a local
+          // run uses (reactive entity, cache reconcile, signals, automation
+          // hooks, transition, notification). Previously ingest only inserted
+          // the row, so a satellite-detected outage fired no notifications or
+          // automations - routing it here is what keeps the two from drifting.
+          const system = await catalogClient
+            .getSystem({ systemId: input.systemId })
+            .catch(() => null);
+          await reactToSatelliteRun({
+            systemId: input.systemId,
+            systemName: system?.name ?? input.systemId,
+            configId: input.configId,
+            configName,
+            environmentId: input.environmentId ?? null,
+            // The env NAME is intentionally omitted (no extra catalog
+            // round-trip): the notifier falls back to the system name.
+            status,
+            latencyMs: input.latencyMs,
+            result: resultRecord,
+            sourceId: input.sourceId,
+            sourceLabel: input.sourceLabel,
+            runTimestamp: new Date(input.executedAt),
+          });
+        } else {
+          // No reactor wired (older host / test harness): fall back to a bare
+          // persist + cache refresh so the run is still recorded. The real host
+          // always wires the reactor above.
+          await service.insertSatelliteRun({
+            configId: input.configId,
+            systemId: input.systemId,
+            environmentId: input.environmentId ?? null,
+            status,
+            latencyMs: input.latencyMs,
+            result: resultRecord,
+            sourceId: input.sourceId,
+            sourceLabel: input.sourceLabel,
+            executedAt: input.executedAt,
+          });
+          await cache.invalidateSystem(input.systemId);
+        }
       },
     ),
 

package/src/satellite-liveness.test.ts

@@ -0,0 +1,199 @@
+import { describe, expect, test } from "bun:test";
+import {
+  buildUnobservableResult,
+  buildUnobservableRun,
+  resolveSatelliteOnlyOutcome,
+} from "./satellite-liveness";
+
+describe("resolveSatelliteOnlyOutcome", () => {
+  test("satellites execute the check when any assigned one is online", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b"],
+        onlineSatelliteIds: ["b"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("records an unobservable run when NO assigned satellite is online", () => {
+    // The bug: this used to return silently, so the check kept displaying its
+    // last status forever and a dead probe read exactly like a passing one.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b"],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("ignores online satellites this check is not assigned to", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a"],
+        onlineSatelliteIds: ["someone-else"],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("stays silent when liveness is UNKNOWN", () => {
+    // A transient failure to reach the satellite service must never mark every
+    // satellite-only check in the fleet degraded at once. Unknown is not the
+    // same as offline, and silence is the pre-existing, safe direction.
+    expect(
+      resolveSatelliteOnlyOutcome({ satelliteIds: ["a"] }),
+    ).toBe("satellites-executing");
+  });
+
+  test("a check with no assigned satellites is treated as executing", () => {
+    // Not reachable from the caller (the branch requires assignments), but the
+    // function must not claim an empty assignment set is unobservable.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: [],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("one online satellite out of many is enough", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "b", "c"],
+        onlineSatelliteIds: ["c"],
+      }),
+    ).toBe("satellites-executing");
+  });
+});
+
+describe("buildUnobservableResult", () => {
+  test("says plainly that health is UNKNOWN, not that the target is down", () => {
+    // The run is degraded because we could not observe, not because the service
+    // failed. The message must not let an operator conclude otherwise.
+    const result = buildUnobservableResult({ satelliteIds: ["a", "b"] });
+
+    expect(String(result.error)).toContain("unknown");
+    expect(String(result.error)).toContain("monitoring gap");
+    expect(String(result.error)).toContain("not a confirmed outage");
+  });
+
+  test("carries a machine-readable marker and the assignment count", () => {
+    const result = buildUnobservableResult({ satelliteIds: ["a", "b"] });
+
+    expect(result.satelliteOffline).toBe(true);
+    expect(result.assignedSatelliteCount).toBe(2);
+  });
+});
+
+describe("buildUnobservableRun", () => {
+  test("records DEGRADED, never unhealthy", () => {
+    // Unhealthy would raise incident-grade alarms about services that may be
+    // perfectly healthy, every time a satellite host reboots.
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] }).status,
+    ).toBe("degraded");
+  });
+
+  test("lands on the slice the job owns, including a concrete environment", () => {
+    // The satellites would have reported for this exact slice, so the gap has
+    // to be recorded there and not on the rollup.
+    expect(
+      buildUnobservableRun({ environmentId: "env-1", satelliteIds: ["a"] })
+        .environmentId,
+    ).toBe("env-1");
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] })
+        .environmentId,
+    ).toBeNull();
+  });
+
+  test("attributes the run to the core, not to a satellite", () => {
+    // A satellite reported nothing - the core is what noticed the gap.
+    expect(
+      buildUnobservableRun({ environmentId: null, satelliteIds: ["a"] })
+        .sourceLabel,
+    ).toBe("Local");
+  });
+
+  test("carries the explanatory result payload", () => {
+    const run = buildUnobservableRun({
+      environmentId: null,
+      satelliteIds: ["a", "b"],
+    });
+
+    expect(run.result.satelliteOffline).toBe(true);
+    expect(run.result.assignedSatelliteCount).toBe(2);
+    expect(String(run.result.error)).toContain("unknown");
+  });
+});
+
+describe("resolveSatelliteOnlyOutcome - assignment changes must not fabricate a gap", () => {
+  /**
+   * The class of bug these guard: an operator changes an assignment and the
+   * platform reacts as though something FAILED. Removing a satellite, adding
+   * local execution back, or swapping one satellite for another are all
+   * deliberate acts - none of them means "nobody is checking this".
+   */
+  test("emptying the satellite list is not an outage", () => {
+    // The executor's branch requires a non-empty list, but the predicate must
+    // agree independently - a future caller must not be able to turn a cleared
+    // assignment into a degraded run.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: [],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("swapping to a different, online satellite is not an outage", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["new-sat"],
+        onlineSatelliteIds: ["new-sat"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("one online satellite is enough even when the others were deleted", () => {
+    // A deleted satellite simply stops appearing in the online set. As long as
+    // ONE assigned satellite is still online, the check is being executed.
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["deleted-sat", "live-sat"],
+        onlineSatelliteIds: ["live-sat", "unrelated-sat"],
+      }),
+    ).toBe("satellites-executing");
+  });
+
+  test("an entirely unrelated fleet being online is NOT enough", () => {
+    // Guards the inverse mistake: "some satellite somewhere is up" must not be
+    // read as "this check is being executed".
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["mine"],
+        onlineSatelliteIds: ["someone-elses", "another"],
+      }),
+    ).toBe("record-unobservable");
+  });
+
+  test("an empty online set with no assignment is not an outage", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({ satelliteIds: [], onlineSatelliteIds: [] }),
+    ).toBe("satellites-executing");
+  });
+
+  test("duplicate ids in an assignment do not change the verdict", () => {
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "a"],
+        onlineSatelliteIds: ["a"],
+      }),
+    ).toBe("satellites-executing");
+    expect(
+      resolveSatelliteOnlyOutcome({
+        satelliteIds: ["a", "a"],
+        onlineSatelliteIds: [],
+      }),
+    ).toBe("record-unobservable");
+  });
+});

package/src/satellite-liveness.ts

@@ -0,0 +1,106 @@
+/**
+ * What a satellite-only check should do when the core reaches its tick.
+ *
+ * ## The bug this exists to close
+ *
+ * A check with `includeLocal: false` and assigned satellites is executed BY
+ * those satellites; the core's own tick has nothing to run and returned
+ * immediately. If every assigned satellite is offline, nobody executes it - and
+ * because the core recorded nothing at all, the check kept displaying whatever
+ * status it last had, indefinitely. A dead probe was indistinguishable from a
+ * passing one, which is the single worst failure mode a monitoring tool can
+ * have.
+ *
+ * So the core now records a `degraded` run instead of staying silent.
+ * `degraded` rather than `unhealthy` because the target may be perfectly
+ * healthy - what failed is our ability to observe it. Marking it unhealthy
+ * would raise incident-grade alarms about services that are fine, every time a
+ * satellite host reboots.
+ */
+export type SatelliteOnlyOutcome =
+  /** Satellites are executing this check; the core has nothing to do. */
+  | "satellites-executing"
+  /** No assigned satellite is online - record a stale run so the gap is visible. */
+  | "record-unobservable";
+
+export function resolveSatelliteOnlyOutcome({
+  satelliteIds,
+  onlineSatelliteIds,
+}: {
+  /** Satellites assigned to this check. */
+  satelliteIds: readonly string[];
+  /**
+   * Currently-online satellite ids. `undefined` when liveness could not be
+   * determined at all (no resolver wired, or the lookup failed).
+   */
+  onlineSatelliteIds?: readonly string[];
+}): SatelliteOnlyOutcome {
+  // Unknown liveness must NEVER manufacture a degraded run: a transient failure
+  // to reach the satellite service would otherwise mark every satellite-only
+  // check degraded across the fleet at once. Staying silent is the pre-existing
+  // behaviour and the safe direction for an unknown.
+  if (onlineSatelliteIds === undefined) return "satellites-executing";
+
+  // An empty assignment set has nothing to be offline. `[].some()` is false, so
+  // without this guard a check with no satellites would be reported as
+  // unobservable - a degraded run for a configuration that cannot produce one.
+  if (satelliteIds.length === 0) return "satellites-executing";
+
+  const online = new Set(onlineSatelliteIds);
+  const anyOnline = satelliteIds.some((id) => online.has(id));
+
+  return anyOnline ? "satellites-executing" : "record-unobservable";
+}
+
+/** The result payload recorded for an unobservable run. */
+export function buildUnobservableResult({
+  satelliteIds,
+}: {
+  satelliteIds: readonly string[];
+}): Record<string, unknown> {
+  const count = satelliteIds.length;
+  return {
+    error:
+      `No assigned satellite is online (${count} assigned), so this check could not be executed. ` +
+      "The target's actual health is unknown - this is a monitoring gap, not a confirmed outage.",
+    satelliteOffline: true,
+    assignedSatelliteCount: count,
+  };
+}
+
+/**
+ * The `persistRunAndReact` arguments for an unobservable run.
+ *
+ * Extracted so the RECORDED VALUES - degraded, the payload's environment slice,
+ * the local source label - are pinned by a test. The executor's mock database
+ * cannot service `persistRunAndReact`'s insert/aggregate chain, so the only way
+ * to assert what gets written is to make the decision about what to write a
+ * separate, pure step.
+ */
+export function buildUnobservableRun({
+  environmentId,
+  satelliteIds,
+}: {
+  /** The single (config, system, env) slice this job owns. */
+  environmentId: string | null;
+  satelliteIds: readonly string[];
+}): {
+  status: "degraded";
+  environmentId: string | null;
+  sourceLabel: string;
+  result: Record<string, unknown>;
+} {
+  return {
+    // Degraded, NOT unhealthy: the target may be perfectly healthy and what
+    // failed is our ability to observe it. Unhealthy would raise
+    // incident-grade alarms about healthy services on every satellite reboot.
+    status: "degraded",
+    // The job payload already names the slice the satellites would have
+    // reported for, so the gap lands exactly where the missing runs would have.
+    environmentId,
+    // Recorded by the CORE, which is what noticed the gap - not by a satellite,
+    // which by definition reported nothing.
+    sourceLabel: "Local",
+    result: buildUnobservableResult({ satelliteIds }),
+  };
+}

package/src/schema.ts

@@ -111,6 +111,27 @@ export const systemHealthChecks = pgTable(
      * `length === 0`; jsonb stores both faithfully.
      */
     environmentIds: jsonb("environment_ids").$type<string[]>(),
+    /**
+     * Per-SATELLITE environment scoping, keyed by satellite id.
+     *
+     * Additive on top of `satelliteIds`, which still answers "which satellites
+     * run this". This answers "and for which environments", so a prod satellite
+     * can run only the prod environment instead of probing every environment
+     * from a network it may have no route to.
+     *
+     * Semantics per entry, mirroring `environmentIds`:
+     * - key ABSENT (or `null`) => that satellite runs every environment the
+     *   assignment resolves to. This is the backfilled default, so existing
+     *   rows keep behaving as they did.
+     * - `[]`       => opt out: that satellite runs ONCE with no environment.
+     * - non-empty  => exactly those ids, INTERSECTED with the assignment's own
+     *   effective set - a satellite can never widen the assignment's scope,
+     *   only narrow it.
+     */
+    satelliteEnvironmentIds:
+      jsonb("satellite_environment_ids").$type<
+        Record<string, string[] | null>
+      >(),
     /**
      * Whether to also run this check locally on the core instance.
      * Defaults to true. Only relevant when satelliteIds is set.

package/src/service-batching.test.ts

@@ -97,7 +97,9 @@ describe("HealthCheckService.getSystemHealthStatus - read batching", () => {
 
     const result = await service.getSystemHealthStatus("system-1");
 
-    expect(result.status).toBe("healthy");
+    // `unknown`, not `healthy`: a system with no checks is unmeasured, and this
+    // test is about transaction batching, not about inventing a status.
+    expect(result.status).toBe("unknown");
     expect(result.checkStatuses).toHaveLength(0);
     const transaction = (mockDb as unknown as { transaction: ReturnType<typeof mock> })
       .transaction;