@checkstack/healthcheck-backend 1.21.3 → 1.23.0

package/src/service-ingest-assertions.test.ts

@@ -13,8 +13,9 @@ import { HealthCheckService } from "./service";
  * Satellite ingest evaluates assertions ON THE CORE (satellites never held
  * the assertion semantics — before this, satellite-executed checks silently
  * skipped assertions), then strips ephemeral fields for parity with local
- * runs. These tests drive `ingestSatelliteResult` against a mock db and
- * assert on the run row it persists.
+ * runs. `processSatelliteResult` returns the processed status + result record
+ * (and the check name); the shared post-run path persists it. These tests
+ * assert on that returned payload.
  */
 
 const KEY = computeAssertionKey({
@@ -30,43 +31,17 @@ const collectorResultSchema = z.object({
   body: healthResultString({ "x-ephemeral": true }),
 });
 
-function buildService({
-  entries,
-  inserted,
-}: {
-  entries: CollectorConfigEntry[];
-  inserted: Record<string, unknown>[];
-}) {
-  const tx = {
-    insert: mock(() => ({
-      values: mock((vals: Record<string, unknown>) => {
-        inserted.push(vals);
-        return Object.assign(Promise.resolve(), {
-          onConflictDoUpdate: mock(() => Promise.resolve()),
-          onConflictDoNothing: mock(() => Promise.resolve()),
-        });
-      }),
-    })),
+function buildService({ entries }: { entries: CollectorConfigEntry[] }) {
+  const db = {
     select: mock(() => ({
       from: mock(() => ({
         where: mock(() =>
-          Object.assign(Promise.resolve([]), {
-            limit: mock(() => Promise.resolve([])),
-          }),
+          Promise.resolve([{ name: "Test check", collectors: entries }]),
         ),
       })),
     })),
   };
 
-  const db = {
-    select: mock(() => ({
-      from: mock(() => ({
-        where: mock(() => Promise.resolve([{ collectors: entries }])),
-      })),
-    })),
-    transaction: mock(async (fn: (t: typeof tx) => Promise<void>) => fn(tx)),
-  };
-
   const collectorRegistry = {
     register: mock(() => {}),
     getCollector: mock(() => ({
@@ -113,64 +88,59 @@ const entries: CollectorConfigEntry[] = [
   },
 ];
 
-async function ingest({
+async function process({
   entries,
   statusCode,
 }: {
   entries: CollectorConfigEntry[];
   statusCode: number;
 }) {
-  const inserted: Record<string, unknown>[] = [];
-  const service = buildService({ entries, inserted });
-  await service.ingestSatelliteResult({
+  const service = buildService({ entries });
+  return service.processSatelliteResult({
     configId: "config-1",
-    systemId: "system-1",
     status: "healthy",
-    latencyMs: 42,
     result: satelliteResult({ statusCode }) as never,
-    executedAt: "2026-07-03T10:00:00.000Z",
-    sourceId: "sat-1",
-    sourceLabel: "EU West",
   });
-  const runInsert = inserted.find((v) => "status" in v && "result" in v);
-  expect(runInsert).toBeDefined();
-  return runInsert as Record<string, unknown>;
 }
 
-function collectorEntryOf(runInsert: Record<string, unknown>) {
-  const result = runInsert.result as {
+function collectorEntryOf(resultRecord: Record<string, unknown>) {
+  const result = resultRecord as {
     metadata: { collectors: Record<string, Record<string, unknown>> };
   };
   return result.metadata.collectors["entry-1"];
 }
 
-describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
+describe("processSatelliteResult - assertion evaluation at ingest", () => {
   it("downgrades a satellite-healthy run whose assertion fails", async () => {
-    const runInsert = await ingest({ entries, statusCode: 404 });
-    expect(runInsert.status).toBe("unhealthy");
+    const { status, resultRecord } = await process({ entries, statusCode: 404 });
+    expect(status).toBe("unhealthy");
 
-    const entry = collectorEntryOf(runInsert);
+    const entry = collectorEntryOf(resultRecord);
     expect(entry._assertionFailed).toBe("statusCode equals 200");
     expect(entry._assertions).toEqual([
       expect.objectContaining({ key: KEY, passed: false, actual: "404" }),
     ]);
-    const message = (runInsert.result as { message: string }).message;
-    expect(message).toBe(
+    expect((resultRecord as { message: string }).message).toBe(
       "Check failed: Assertion failed: statusCode equals 200",
     );
   });
 
   it("keeps a passing run healthy and stores the passing outcome", async () => {
-    const runInsert = await ingest({ entries, statusCode: 200 });
-    expect(runInsert.status).toBe("healthy");
+    const { status, resultRecord } = await process({ entries, statusCode: 200 });
+    expect(status).toBe("healthy");
 
-    const entry = collectorEntryOf(runInsert);
+    const entry = collectorEntryOf(resultRecord);
     expect(entry._assertionFailed).toBeUndefined();
     expect(entry._assertions).toEqual([
       expect.objectContaining({ key: KEY, passed: true, actual: "200" }),
     ]);
   });
 
+  it("resolves the check name for the notification", async () => {
+    const { configName } = await process({ entries, statusCode: 200 });
+    expect(configName).toBe("Test check");
+  });
+
   it("strips ephemeral fields AFTER assertions ran against them", async () => {
     const withBodyAssertion: CollectorConfigEntry[] = [
       {
@@ -187,12 +157,12 @@ describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
         ],
       },
     ];
-    const runInsert = await ingest({
+    const { status, resultRecord } = await process({
       entries: withBodyAssertion,
       statusCode: 200,
     });
 
-    const entry = collectorEntryOf(runInsert);
+    const entry = collectorEntryOf(resultRecord);
     // The JSONPath assertion evaluated against the (ephemeral) body...
     expect(entry._assertions).toEqual([
       expect.objectContaining({ passed: true, actual: "ok" }),
@@ -200,14 +170,17 @@ describe("ingestSatelliteResult - assertion evaluation at ingest", () => {
     // ...but the body itself never reaches storage.
     expect(entry.body).toBeUndefined();
     expect(entry.statusCode).toBe(200);
-    expect(runInsert.status).toBe("healthy");
+    expect(status).toBe("healthy");
   });
 
   it("tolerates collector entries the config no longer knows", async () => {
-    const runInsert = await ingest({ entries: [], statusCode: 500 });
+    const { status, resultRecord } = await process({
+      entries: [],
+      statusCode: 500,
+    });
     // No assertions configured: status passes through untouched.
-    expect(runInsert.status).toBe("healthy");
-    const entry = collectorEntryOf(runInsert);
+    expect(status).toBe("healthy");
+    const entry = collectorEntryOf(resultRecord);
     expect(entry._assertions).toBeUndefined();
   });
 });

package/src/service-paused-filter.test.ts

@@ -159,9 +159,12 @@ describe("HealthCheckService - paused configuration filtering", () => {
       const result = await service.getSystemHealthStatus("system-1");
 
       // The post-filter associations list is empty, so the system has no
-      // active checks and reads healthy — paused failures do NOT keep the
-      // system degraded.
-      expect(result.status).toBe("healthy");
+      // ACTIVE checks. Paused failures still do NOT keep the system degraded -
+      // that is the behaviour this test guards - but the result is now
+      // `unknown` rather than `healthy`: with its only check paused, nothing is
+      // measuring this system, and claiming health it has no evidence for is
+      // what made a broken check read green on the catalog and status page.
+      expect(result.status).toBe("unknown");
       expect(result.checkStatuses).toHaveLength(0);
     });
 
@@ -260,7 +263,9 @@ describe("HealthCheckService - paused configuration filtering", () => {
 
       const result = await service.getSystemHealthStatus("system-1");
 
-      expect(result.status).toBe("healthy");
+      // No enabled associations = nothing measured, so `unknown` rather than an
+      // invented `healthy`.
+      expect(result.status).toBe("unknown");
       expect(result.checkStatuses).toHaveLength(0);
     });
   });

package/src/service-rollup-worst-wins.test.ts

@@ -48,18 +48,46 @@ type Run = { status: "healthy" | "degraded" | "unhealthy"; timestamp: Date };
 
 /**
  * Build a mock db for the rollup path. `runsByEnv` maps each environment key
- * (`null` = env-less) to that env's runs (DESC). `environmentIds` is the
- * assignment's selector under test. The per-env runs query resolves against the
- * concrete env id bound in its predicate (or the env-less slice when none of the
- * known env ids appear, i.e. the `isNull` clause).
+ * (`null` = env-less) to that env's runs (DESC), all attributed to the LOCAL
+ * core; `satelliteRuns` adds slices probed from a satellite. `environmentIds` is
+ * the assignment's selector under test.
+ *
+ * The per-slice runs query resolves against the env id AND source id bound in
+ * its predicate. Known env ids and satellite ids are disjoint in these fixtures,
+ * so each is recovered by set membership; an absent value means the `isNull`
+ * clause, i.e. the env-less / local slice.
  */
 function createRollupMockDb(props: {
   runsByEnv: Map<string | null, Run[]>;
   environmentIds: string[] | null;
+  satelliteRuns?: { sourceId: string; environmentId: string | null; runs: Run[] }[];
+  satelliteIds?: string[] | null;
+  satelliteEnvironmentIds?: Record<string, string[] | null> | null;
+  includeLocal?: boolean;
 }) {
-  const { runsByEnv, environmentIds } = props;
+  const {
+    runsByEnv,
+    environmentIds,
+    satelliteRuns = [],
+    satelliteIds = null,
+    satelliteEnvironmentIds = null,
+    includeLocal = true,
+  } = props;
+  const slices = [
+    ...[...runsByEnv].map(([environmentId, runs]) => ({
+      environmentId,
+      sourceId: null as string | null,
+      runs,
+    })),
+    ...satelliteRuns,
+  ];
   const knownEnvIds = new Set(
-    [...runsByEnv.keys()].filter((k): k is string => k !== null),
+    slices
+      .map((s) => s.environmentId)
+      .filter((k): k is string => k !== null),
+  );
+  const knownSourceIds = new Set(
+    slices.map((s) => s.sourceId).filter((k): k is string => k !== null),
   );
 
   const assocWhere = mock(() =>
@@ -71,6 +99,9 @@ function createRollupMockDb(props: {
         paused: false,
         stateThresholds: null,
         environmentIds,
+        satelliteIds,
+        satelliteEnvironmentIds,
+        includeLocal,
       },
     ]),
   );
@@ -81,25 +112,35 @@ function createRollupMockDb(props: {
     innerJoin: mock(() => assocInnerJoin),
   });
 
-  // Per-env runs query: pick the slice named by the predicate's env value.
-  const resolvePerEnv = (predicate: unknown): Run[] => {
+  // Per-slice runs query: pick the slice named by the predicate's env + source.
+  const resolveSlice = (predicate: unknown): Run[] => {
     const values = collectPredicateValues(predicate);
-    const envId = values.find((v) => knownEnvIds.has(v)) ?? null;
-    return runsByEnv.get(envId) ?? [];
+    const environmentId = values.find((v) => knownEnvIds.has(v)) ?? null;
+    const sourceId = values.find((v) => knownSourceIds.has(v)) ?? null;
+    return (
+      slices.find(
+        (s) => s.environmentId === environmentId && s.sourceId === sourceId,
+      )?.runs ?? []
+    );
   };
   const runsFromFor = () => {
     const runsWhere = mock((predicate: unknown) => {
-      const rows = resolvePerEnv(predicate);
+      const rows = resolveSlice(predicate);
       const limit = mock(() => Promise.resolve(rows));
       return { orderBy: mock(() => ({ limit })), limit };
     });
     return Object.assign(Promise.resolve([]), { where: runsWhere });
   };
 
-  // Distinct env keys query: select({environmentId}).from().where().
+  // Distinct slice keys query: selectDistinct({environmentId, sourceId}).
   const distinctFrom = Object.assign(Promise.resolve([]), {
     where: mock(() =>
-      Promise.resolve([...runsByEnv.keys()].map((k) => ({ environmentId: k }))),
+      Promise.resolve(
+        slices.map(({ environmentId, sourceId }) => ({
+          environmentId,
+          sourceId,
+        })),
+      ),
     ),
   });
 
@@ -245,58 +286,109 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
     expect(result.checkStatuses[0].runsConsidered).toBe(3);
   });
 
-  it("the per-env slice (concrete environmentId) is unaffected — only the rollup branch changed", async () => {
-    // Pass an explicit environmentId; the per-env branch (string envId) still
-    // filters to that env's slice via a single windowed query and reads unhealthy.
-    const prodOnly = runs("unhealthy", 5);
-    const assocWhere = mock(() =>
-      Promise.resolve([
-        {
-          configurationId: "config-1",
-          configName: "HTTP probe",
-          enabled: true,
-          paused: false,
-          stateThresholds: null,
-          environmentIds: null,
-        },
-      ]),
-    );
-    const assocInnerJoin = Object.assign(Promise.resolve([]), { where: assocWhere });
-    const assocFrom = Object.assign(Promise.resolve([]), {
-      innerJoin: mock(() => assocInnerJoin),
+  it("reports unhealthy when the LOCAL check passes but a SATELLITE check fails", async () => {
+    // The reported bug (@stuajnht): a system read HEALTHY while one of its
+    // probe locations failed every single time. Both sources' runs landed in
+    // one slice, so `evaluateConsecutive` saw healthy/unhealthy alternating,
+    // broke its streak on every run, met no threshold, and fell through to its
+    // healthy default. Sliced per source, the satellite is evaluated on its own
+    // and worst-wins carries it to the system.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
 
-    const runsLimit = mock(() => Promise.resolve(prodOnly));
-    const runsOrderBy = mock(() => ({ limit: runsLimit }));
-    const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
-    const runsFrom = Object.assign(Promise.resolve(prodOnly), {
-      where: runsWhere,
-      orderBy: runsOrderBy,
+    expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].status).toBe("unhealthy");
+    // Two locations probing one environment = two slices, one failing.
+    expect(result.checkStatuses[0].sliceCount).toBe(2);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+  });
+
+  it("names the failing location in the per-slice breakdown", async () => {
+    const runsByEnv = new Map<string | null, Run[]>([["prod", runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: ["prod"],
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: "prod", runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
 
-    let selectCallCount = 0;
-    const mockDb = withTransactionMock({
-      select: mock(() => {
-        selectCallCount += 1;
-        if (selectCallCount === 1) return { from: mock(() => assocFrom) };
-        return { from: mock(() => runsFrom) };
-      }),
-      insert: mock(() => ({
-        values: mock(() => ({
-          onConflictDoUpdate: mock(() => Promise.resolve()),
-          onConflictDoNothing: mock(() => Promise.resolve()),
-          returning: mock(() => Promise.resolve([])),
-        })),
-      })),
-      update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
-      delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
-      execute: mock(() => Promise.resolve()),
+    const { slices } = (await service.getSystemHealthStatus("system-1")).checkStatuses[0];
+
+    expect(slices).toHaveLength(2);
+    expect(slices.find((s) => s.sourceId === null)?.status).toBe("healthy");
+    expect(slices.find((s) => s.sourceId === "sat-eu")?.status).toBe("unhealthy");
+  });
+
+  it("drops a de-assigned satellite's stale failing runs", async () => {
+    // No health-change event fires for a slice that merely stopped producing
+    // runs, so without the effective-source filter an unassigned satellite's
+    // last failures would drag the rollup until they aged out of the window.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      satelliteIds: [],
+      satelliteRuns: [
+        { sourceId: "sat-gone", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
+
+    expect(result.status).toBe("healthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+  });
+
+  it("keeps evaluating satellites when the core stopped running the check", async () => {
+    // `includeLocal: false` means the core's old runs no longer contribute, so
+    // the satellite alone decides.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      includeLocal: false,
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
+    });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
 
+    expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+  });
+
+  it("a pinned environment evaluates only that environment's slices", async () => {
+    // Passing an explicit environmentId narrows to that env - the other env's
+    // runs must not contribute - while the source dimension is still sliced.
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
     const result = await service.getSystemHealthStatus("system-1", "prod");
     expect(result.status).toBe("unhealthy");
     expect(result.checkStatuses[0].sliceCount).toBe(1);
     expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+    expect(result.checkStatuses[0].runsConsidered).toBe(5);
   });
 });