@checkstack/healthcheck-backend 1.21.2 → 1.22.0

package/src/service-rollup-worst-wins.test.ts

@@ -48,18 +48,46 @@ type Run = { status: "healthy" | "degraded" | "unhealthy"; timestamp: Date };
 
 /**
  * Build a mock db for the rollup path. `runsByEnv` maps each environment key
- * (`null` = env-less) to that env's runs (DESC). `environmentIds` is the
- * assignment's selector under test. The per-env runs query resolves against the
- * concrete env id bound in its predicate (or the env-less slice when none of the
- * known env ids appear, i.e. the `isNull` clause).
+ * (`null` = env-less) to that env's runs (DESC), all attributed to the LOCAL
+ * core; `satelliteRuns` adds slices probed from a satellite. `environmentIds` is
+ * the assignment's selector under test.
+ *
+ * The per-slice runs query resolves against the env id AND source id bound in
+ * its predicate. Known env ids and satellite ids are disjoint in these fixtures,
+ * so each is recovered by set membership; an absent value means the `isNull`
+ * clause, i.e. the env-less / local slice.
  */
 function createRollupMockDb(props: {
   runsByEnv: Map<string | null, Run[]>;
   environmentIds: string[] | null;
+  satelliteRuns?: { sourceId: string; environmentId: string | null; runs: Run[] }[];
+  satelliteIds?: string[] | null;
+  satelliteEnvironmentIds?: Record<string, string[] | null> | null;
+  includeLocal?: boolean;
 }) {
-  const { runsByEnv, environmentIds } = props;
+  const {
+    runsByEnv,
+    environmentIds,
+    satelliteRuns = [],
+    satelliteIds = null,
+    satelliteEnvironmentIds = null,
+    includeLocal = true,
+  } = props;
+  const slices = [
+    ...[...runsByEnv].map(([environmentId, runs]) => ({
+      environmentId,
+      sourceId: null as string | null,
+      runs,
+    })),
+    ...satelliteRuns,
+  ];
   const knownEnvIds = new Set(
-    [...runsByEnv.keys()].filter((k): k is string => k !== null),
+    slices
+      .map((s) => s.environmentId)
+      .filter((k): k is string => k !== null),
+  );
+  const knownSourceIds = new Set(
+    slices.map((s) => s.sourceId).filter((k): k is string => k !== null),
   );
 
   const assocWhere = mock(() =>
@@ -71,6 +99,9 @@ function createRollupMockDb(props: {
         paused: false,
         stateThresholds: null,
         environmentIds,
+        satelliteIds,
+        satelliteEnvironmentIds,
+        includeLocal,
       },
     ]),
   );
@@ -81,25 +112,35 @@ function createRollupMockDb(props: {
     innerJoin: mock(() => assocInnerJoin),
   });
 
-  // Per-env runs query: pick the slice named by the predicate's env value.
-  const resolvePerEnv = (predicate: unknown): Run[] => {
+  // Per-slice runs query: pick the slice named by the predicate's env + source.
+  const resolveSlice = (predicate: unknown): Run[] => {
     const values = collectPredicateValues(predicate);
-    const envId = values.find((v) => knownEnvIds.has(v)) ?? null;
-    return runsByEnv.get(envId) ?? [];
+    const environmentId = values.find((v) => knownEnvIds.has(v)) ?? null;
+    const sourceId = values.find((v) => knownSourceIds.has(v)) ?? null;
+    return (
+      slices.find(
+        (s) => s.environmentId === environmentId && s.sourceId === sourceId,
+      )?.runs ?? []
+    );
   };
   const runsFromFor = () => {
     const runsWhere = mock((predicate: unknown) => {
-      const rows = resolvePerEnv(predicate);
+      const rows = resolveSlice(predicate);
       const limit = mock(() => Promise.resolve(rows));
       return { orderBy: mock(() => ({ limit })), limit };
     });
     return Object.assign(Promise.resolve([]), { where: runsWhere });
   };
 
-  // Distinct env keys query: select({environmentId}).from().where().
+  // Distinct slice keys query: selectDistinct({environmentId, sourceId}).
   const distinctFrom = Object.assign(Promise.resolve([]), {
     where: mock(() =>
-      Promise.resolve([...runsByEnv.keys()].map((k) => ({ environmentId: k }))),
+      Promise.resolve(
+        slices.map(({ environmentId, sourceId }) => ({
+          environmentId,
+          sourceId,
+        })),
+      ),
     ),
   });
 
@@ -245,58 +286,109 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
     expect(result.checkStatuses[0].runsConsidered).toBe(3);
   });
 
-  it("the per-env slice (concrete environmentId) is unaffected — only the rollup branch changed", async () => {
-    // Pass an explicit environmentId; the per-env branch (string envId) still
-    // filters to that env's slice via a single windowed query and reads unhealthy.
-    const prodOnly = runs("unhealthy", 5);
-    const assocWhere = mock(() =>
-      Promise.resolve([
-        {
-          configurationId: "config-1",
-          configName: "HTTP probe",
-          enabled: true,
-          paused: false,
-          stateThresholds: null,
-          environmentIds: null,
-        },
-      ]),
-    );
-    const assocInnerJoin = Object.assign(Promise.resolve([]), { where: assocWhere });
-    const assocFrom = Object.assign(Promise.resolve([]), {
-      innerJoin: mock(() => assocInnerJoin),
+  it("reports unhealthy when the LOCAL check passes but a SATELLITE check fails", async () => {
+    // The reported bug (@stuajnht): a system read HEALTHY while one of its
+    // probe locations failed every single time. Both sources' runs landed in
+    // one slice, so `evaluateConsecutive` saw healthy/unhealthy alternating,
+    // broke its streak on every run, met no threshold, and fell through to its
+    // healthy default. Sliced per source, the satellite is evaluated on its own
+    // and worst-wins carries it to the system.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
 
-    const runsLimit = mock(() => Promise.resolve(prodOnly));
-    const runsOrderBy = mock(() => ({ limit: runsLimit }));
-    const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
-    const runsFrom = Object.assign(Promise.resolve(prodOnly), {
-      where: runsWhere,
-      orderBy: runsOrderBy,
+    expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].status).toBe("unhealthy");
+    // Two locations probing one environment = two slices, one failing.
+    expect(result.checkStatuses[0].sliceCount).toBe(2);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+  });
+
+  it("names the failing location in the per-slice breakdown", async () => {
+    const runsByEnv = new Map<string | null, Run[]>([["prod", runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: ["prod"],
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: "prod", runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
 
-    let selectCallCount = 0;
-    const mockDb = withTransactionMock({
-      select: mock(() => {
-        selectCallCount += 1;
-        if (selectCallCount === 1) return { from: mock(() => assocFrom) };
-        return { from: mock(() => runsFrom) };
-      }),
-      insert: mock(() => ({
-        values: mock(() => ({
-          onConflictDoUpdate: mock(() => Promise.resolve()),
-          onConflictDoNothing: mock(() => Promise.resolve()),
-          returning: mock(() => Promise.resolve([])),
-        })),
-      })),
-      update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
-      delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
-      execute: mock(() => Promise.resolve()),
+    const { slices } = (await service.getSystemHealthStatus("system-1")).checkStatuses[0];
+
+    expect(slices).toHaveLength(2);
+    expect(slices.find((s) => s.sourceId === null)?.status).toBe("healthy");
+    expect(slices.find((s) => s.sourceId === "sat-eu")?.status).toBe("unhealthy");
+  });
+
+  it("drops a de-assigned satellite's stale failing runs", async () => {
+    // No health-change event fires for a slice that merely stopped producing
+    // runs, so without the effective-source filter an unassigned satellite's
+    // last failures would drag the rollup until they aged out of the window.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      satelliteIds: [],
+      satelliteRuns: [
+        { sourceId: "sat-gone", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
     });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
+
+    expect(result.status).toBe("healthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+  });
+
+  it("keeps evaluating satellites when the core stopped running the check", async () => {
+    // `includeLocal: false` means the core's old runs no longer contribute, so
+    // the satellite alone decides.
+    const runsByEnv = new Map<string | null, Run[]>([[null, runs("healthy", 5)]]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: null,
+      includeLocal: false,
+      satelliteIds: ["sat-eu"],
+      satelliteRuns: [
+        { sourceId: "sat-eu", environmentId: null, runs: runs("unhealthy", 5) },
+      ],
+    });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
 
+    expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+  });
+
+  it("a pinned environment evaluates only that environment's slices", async () => {
+    // Passing an explicit environmentId narrows to that env - the other env's
+    // runs must not contribute - while the source dimension is still sliced.
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
     const result = await service.getSystemHealthStatus("system-1", "prod");
     expect(result.status).toBe("unhealthy");
     expect(result.checkStatuses[0].sliceCount).toBe(1);
     expect(result.checkStatuses[0].failingSliceCount).toBe(1);
+    expect(result.checkStatuses[0].runsConsidered).toBe(5);
   });
 });