@checkstack/healthcheck-backend 1.17.0 → 1.19.0

package/src/service-rollup-worst-wins.test.ts

@@ -1,98 +1,146 @@
 import { describe, it, expect, mock } from "bun:test";
+import { withTransactionMock } from "@checkstack/test-utils-backend";
 import { HealthCheckService } from "./service";
 import { evaluateHealthStatus } from "./state-evaluator";
 
 /**
- * Regression coverage for the system-rollup worst-wins-across-environments
- * fix in `getSystemHealthStatus(systemId)` (the `environmentId === undefined`
- * branch).
+ * Regression coverage for the system-rollup derivation in
+ * `getSystemHealthStatus(systemId)` (the `environmentId === undefined` branch):
  *
- * The original branch flattened every environment's runs into one
- * `timestamp DESC` list and handed the interleaved list to the threshold
- * evaluator (the default `consecutive` mode). Consecutive mode walks
- * newest-first and breaks the streak on the first interleaving env, so the
- * rollup collapsed to whichever env ran last in the batch — masking any
- * permanently-failing sibling env ("the healthy env wins" / latest-wins)
- * and flapping whenever env insertion order drifted across ticks.
+ *  1. Worst-wins ACROSS environments within an association. The original branch
+ *     flattened every environment's runs into one `timestamp DESC` list and
+ *     handed the interleaved list to the threshold evaluator (default
+ *     `consecutive` mode). Consecutive mode walks newest-first and breaks the
+ *     streak on the first interleaving env, so the rollup collapsed to whichever
+ *     env ran last — masking a permanently-failing sibling env ("the healthy env
+ *     wins" / latest-wins) and flapping whenever env insertion order drifted.
+ *     The fix evaluates a FULL per-env window and takes worst-wins across envs.
  *
- * The fix evaluates the threshold window PER ENVIRONMENT within the
- * association and takes worst-wins across envs (unhealthy > degraded >
- * healthy), making the rollup stable regardless of insertion order or
- * multi-pod racing. These tests pin that behavior with a mocked DB that
- * returns the interleaved mixed-pool the real query would surface.
+ *  2. Currently-effective-slice filtering. A per-env slice whose environment was
+ *     DISABLED for the assignment (removed from `environmentIds`) must STOP
+ *     contributing immediately - its stale unhealthy runs must not keep dragging
+ *     the rollup until they age out of the window.
+ *
+ * Each environment is now windowed by its OWN query (per-env `LIMIT`), so the
+ * mock resolves each per-env runs query against the env bound in its predicate.
  */
-describe("HealthCheckService - system rollup worst-wins across environments", () => {
-  /**
-   * The mixed-pool query captured by the mock. Ordered DESC (newest first),
-   * exactly the shape the real `health_check_runs` query returns. Two envs
-   * (`prod`, `staging`) of one assignment, both fanning out every tick, prod
-   * permanently unhealthy and staging permanently healthy. The env insertion
-   * order in the executor is sequential membership order, so prod lands before
-   * staging, making the latest run in the pool a staging-healthy run — the
-   * exact scenario that masked prod's outage under flattening.
-   */
-  const PROD_RUN = { status: "unhealthy" as const, environmentId: "prod" };
-  const STAGE_RUN = { status: "healthy" as const, environmentId: "staging" };
-
-  function buildMixedPool(ticksPerEnv = 5): { status: "unhealthy" | "healthy"; timestamp: Date; environmentId: string }[] {
-    const pool: { status: "unhealthy" | "healthy"; timestamp: Date; environmentId: string }[] = [];
-    for (let i = 0; i < ticksPerEnv; i++) {
-      pool.push({ ...PROD_RUN, timestamp: new Date(2025, 0, 1, 0, 0, i) });
-      pool.push({ ...STAGE_RUN, timestamp: new Date(2025, 0, 1, 0, 0, i + 0.5) });
+
+/** Walk a drizzle predicate object and collect every bound literal value. */
+function collectPredicateValues(predicate: unknown): string[] {
+  const values: string[] = [];
+  const seen = new Set<unknown>();
+  const walk = (node: unknown) => {
+    if (node == null || seen.has(node) || typeof node !== "object") return;
+    seen.add(node);
+    if ("value" in (node as Record<string, unknown>)) {
+      const v = (node as { value: unknown }).value;
+      if (typeof v === "string") values.push(v);
     }
-    return pool; // DESC at the DB layer; we return newest-first below.
-  }
+    for (const child of Object.values(node as Record<string, unknown>)) {
+      walk(child);
+    }
+  };
+  walk(predicate);
+  return values;
+}
+
+type Run = { status: "healthy" | "degraded" | "unhealthy"; timestamp: Date };
 
-  function createMockDb(runsMixedDesc: { status: string; timestamp: Date; environmentId: string }[]) {
-    const assocWhere = mock(() => Promise.resolve([
+/**
+ * Build a mock db for the rollup path. `runsByEnv` maps each environment key
+ * (`null` = env-less) to that env's runs (DESC). `environmentIds` is the
+ * assignment's selector under test. The per-env runs query resolves against the
+ * concrete env id bound in its predicate (or the env-less slice when none of the
+ * known env ids appear, i.e. the `isNull` clause).
+ */
+function createRollupMockDb(props: {
+  runsByEnv: Map<string | null, Run[]>;
+  environmentIds: string[] | null;
+}) {
+  const { runsByEnv, environmentIds } = props;
+  const knownEnvIds = new Set(
+    [...runsByEnv.keys()].filter((k): k is string => k !== null),
+  );
+
+  const assocWhere = mock(() =>
+    Promise.resolve([
       {
         configurationId: "config-1",
         configName: "HTTP probe",
         enabled: true,
         paused: false,
         stateThresholds: null,
+        environmentIds,
       },
-    ]));
-    const assocInnerJoin = Object.assign(Promise.resolve([]), { where: assocWhere });
-    const assocFrom = Object.assign(Promise.resolve([]), { innerJoin: mock(() => assocInnerJoin) });
+    ]),
+  );
+  const assocInnerJoin = Object.assign(Promise.resolve([]), {
+    where: assocWhere,
+  });
+  const assocFrom = Object.assign(Promise.resolve([]), {
+    innerJoin: mock(() => assocInnerJoin),
+  });
 
-    const runsLimit = mock(() => Promise.resolve(runsMixedDesc));
-    const runsOrderBy = mock(() => ({ limit: runsLimit }));
-    const runsWhere = mock(() => ({ orderBy: runsOrderBy, limit: runsLimit }));
-    const runsFrom = Object.assign(Promise.resolve(runsMixedDesc), {
-      where: runsWhere,
-      orderBy: runsOrderBy,
+  // Per-env runs query: pick the slice named by the predicate's env value.
+  const resolvePerEnv = (predicate: unknown): Run[] => {
+    const values = collectPredicateValues(predicate);
+    const envId = values.find((v) => knownEnvIds.has(v)) ?? null;
+    return runsByEnv.get(envId) ?? [];
+  };
+  const runsFromFor = () => {
+    const runsWhere = mock((predicate: unknown) => {
+      const rows = resolvePerEnv(predicate);
+      const limit = mock(() => Promise.resolve(rows));
+      return { orderBy: mock(() => ({ limit })), limit };
     });
+    return Object.assign(Promise.resolve([]), { where: runsWhere });
+  };
 
-    let selectCallCount = 0;
-    return {
-      select: mock(() => {
-        selectCallCount += 1;
-        if (selectCallCount === 1) return { from: mock(() => assocFrom) };
-        return { from: mock(() => runsFrom) };
-      }),
-      insert: mock(() => ({
-        values: mock(() => ({
-          onConflictDoUpdate: mock(() => Promise.resolve()),
-          onConflictDoNothing: mock(() => Promise.resolve()),
-          returning: mock(() => Promise.resolve([])),
-        })),
+  // Distinct env keys query: select({environmentId}).from().where().
+  const distinctFrom = Object.assign(Promise.resolve([]), {
+    where: mock(() =>
+      Promise.resolve([...runsByEnv.keys()].map((k) => ({ environmentId: k }))),
+    ),
+  });
+
+  let selectCallCount = 0;
+  return withTransactionMock({
+    select: mock(() => {
+      selectCallCount += 1;
+      // #1 associations; every subsequent select is a per-env runs window.
+      if (selectCallCount === 1) return { from: mock(() => assocFrom) };
+      return { from: mock(() => runsFromFor()) };
+    }),
+    selectDistinct: mock(() => ({ from: mock(() => distinctFrom) })),
+    insert: mock(() => ({
+      values: mock(() => ({
+        onConflictDoUpdate: mock(() => Promise.resolve()),
+        onConflictDoNothing: mock(() => Promise.resolve()),
+        returning: mock(() => Promise.resolve([])),
       })),
-      update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
-      delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
-      execute: mock(() => Promise.resolve()),
-    };
-  }
-
-  it("the rollup reports unhealthy when ONE env is permanently unhealthy, the other healthy", async () => {
-    // DB returns newest-first interleaved runs. The pre-fix behavior would
-    // mask prod's outage because the latest run is staging-healthy; the
-    // threshold evaluator (default consecutive mode) walks newest-first from
-    // staging-healthy, breaks the streak on the very next prod-unhealthy run,
-    // and falls back to `"healthy"`.
-    const pool = buildMixedPool(5);
-    const runsDesc = pool.toReversed(); // oldest produced first above; reverse to DESC
-    const mockDb = createMockDb(runsDesc as never);
+    })),
+    update: mock(() => ({
+      set: mock(() => ({ where: mock(() => Promise.resolve()) })),
+    })),
+    delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
+    execute: mock(() => Promise.resolve()),
+  });
+}
+
+function runs(status: Run["status"], count: number, envSecondOffset = 0): Run[] {
+  return Array.from({ length: count }, (_, i) => ({
+    status,
+    timestamp: new Date(2025, 0, 1, 0, 0, i, envSecondOffset),
+  })).toReversed(); // DESC (newest first)
+}
+
+describe("HealthCheckService - system rollup worst-wins across environments", () => {
+  it("reports unhealthy when ONE env is permanently unhealthy, the other healthy", async () => {
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
 
     const result = await service.getSystemHealthStatus("system-1");
@@ -100,29 +148,46 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
     expect(result.status).toBe("unhealthy");
     expect(result.checkStatuses).toHaveLength(1);
     expect(result.checkStatuses[0].status).toBe("unhealthy");
-    expect(result.checkStatuses[0].runsConsidered).toBe(pool.length);
+    expect(result.checkStatuses[0].runsConsidered).toBe(10);
+    // Fan-out accounting: two environment slices (prod + staging), one failing.
+    expect(result.checkStatuses[0].sliceCount).toBe(2);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
   });
 
-  it("flattening the same mixed pool through the evaluator (the pre-fix derivation) would have returned `healthy`", async () => {
+  it("counts every failing environment slice for the fan-out denominator (3 envs, 2 failing)", async () => {
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      ["eu", runs("unhealthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
+    expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(3);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(2);
+  });
+
+  it("flattening the same mixed pool through the evaluator (the pre-fix derivation) would have returned `healthy`", () => {
     // Sanity check: the very data the rollup branch reads, fed directly to
     // `evaluateHealthStatus` as one flat interleaved list, collapses to
-    // "healthy" — the precise regression this fix replaces with per-env
-    // evaluation. Pinning it here guards against a relax of the test above.
-    const pool = buildMixedPool(5);
-    const runsDesc = pool.toReversed();
-    const flatStatus = evaluateHealthStatus({
-      runs: runsDesc as never,
-    });
+    // "healthy" — the precise regression per-env evaluation replaces.
+    const pool: Run[] = [];
+    for (let i = 0; i < 5; i++) {
+      pool.push({ status: "unhealthy", timestamp: new Date(2025, 0, 1, 0, 0, i) });
+      pool.push({ status: "healthy", timestamp: new Date(2025, 0, 1, 0, 0, i, 500) });
+    }
+    const flatStatus = evaluateHealthStatus({ runs: pool.toReversed() as never });
     expect(flatStatus).toBe("healthy");
   });
 
   it("reports healthy only when EVERY env is healthy", async () => {
-    const allHealthy = Array.from({ length: 10 }, (_, i) => ({
-      status: "healthy",
-      timestamp: new Date(2025, 0, 1, 0, 0, i),
-      environmentId: i % 2 === 0 ? "prod" : "staging",
-    })).toReversed();
-    const mockDb = createMockDb(allHealthy as never);
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("healthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
 
     const result = await service.getSystemHealthStatus("system-1");
@@ -130,46 +195,76 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
   });
 
   it("degrades (not flaps) when one env is degraded and the other healthy", async () => {
-    // Default consecutive thresholds need 2 consecutive failures to escalate
-    // to `degraded` (and 5 to escalate to `unhealthy` — so keep prod's streak
-    // at exactly 2 degraded runs). Per-env: prod's env-sorted slice =
-    // [degraded, degraded] (newest first) → degraded; staging → healthy.
-    // Rollup worst-wins = degraded. Flattening would break on the staging
-    // interleave and return `healthy` (the masked bug); per-env gives a
-    // stable `degraded`.
-    const pool: { status: "healthy" | "degraded"; timestamp: Date; environmentId: string }[] = [];
-    for (let i = 0; i < 2; i++) {
-      pool.push({ status: "degraded", timestamp: new Date(2025, 0, 1, 0, 0, i), environmentId: "prod" });
-      pool.push({ status: "healthy", timestamp: new Date(2025, 0, 1, 0, 0, i + 0.5), environmentId: "staging" });
-    }
-    const runsDesc = pool.toReversed();
-    const mockDb = createMockDb(runsDesc as never);
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("degraded", 2)],
+      ["staging", runs("healthy", 2)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: null });
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
 
     const result = await service.getSystemHealthStatus("system-1");
     expect(result.status).toBe("degraded");
   });
 
+  it("drops a DISABLED environment's stale unhealthy runs from the rollup (regression)", async () => {
+    // prod was DISABLED for the assignment (environmentIds now ['staging']) but
+    // its historical unhealthy runs still exist. The rollup must ignore prod and
+    // read healthy from the sole effective env (staging), immediately - not after
+    // prod's runs age out of the window.
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      ["staging", runs("healthy", 5)],
+    ]);
+    const mockDb = createRollupMockDb({
+      runsByEnv,
+      environmentIds: ["staging"],
+    });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
+
+    expect(result.status).toBe("healthy");
+    expect(result.checkStatuses[0].status).toBe("healthy");
+    // Only the effective (staging) slice counts now.
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(0);
+    expect(result.checkStatuses[0].runsConsidered).toBe(5);
+  });
+
+  it("opting out ([]) drops all concrete-env runs and keeps only the env-less slice", async () => {
+    const runsByEnv = new Map<string | null, Run[]>([
+      ["prod", runs("unhealthy", 5)],
+      [null, runs("healthy", 3)],
+    ]);
+    const mockDb = createRollupMockDb({ runsByEnv, environmentIds: [] });
+    const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
+
+    const result = await service.getSystemHealthStatus("system-1");
+    expect(result.status).toBe("healthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+    expect(result.checkStatuses[0].runsConsidered).toBe(3);
+  });
+
   it("the per-env slice (concrete environmentId) is unaffected — only the rollup branch changed", async () => {
-    // Pass an explicit environmentId; the old per-env branch (string envId)
-    // must continue to filter to that env's slice. Here we ask for `prod`
-    // and expect unhealthy.
-    const prodOnly = Array.from({ length: 5 }, (_, i) => ({
-      status: "unhealthy",
-      timestamp: new Date(2025, 0, 1, 0, 0, i),
-    }));
-    // Mock: the runs query mirrors the predicate back to prodOnly.
-    const assocWhere = mock(() => Promise.resolve([
-      {
-        configurationId: "config-1",
-        configName: "HTTP probe",
-        enabled: true,
-        paused: false,
-        stateThresholds: null,
-      },
-    ]));
+    // Pass an explicit environmentId; the per-env branch (string envId) still
+    // filters to that env's slice via a single windowed query and reads unhealthy.
+    const prodOnly = runs("unhealthy", 5);
+    const assocWhere = mock(() =>
+      Promise.resolve([
+        {
+          configurationId: "config-1",
+          configName: "HTTP probe",
+          enabled: true,
+          paused: false,
+          stateThresholds: null,
+          environmentIds: null,
+        },
+      ]),
+    );
     const assocInnerJoin = Object.assign(Promise.resolve([]), { where: assocWhere });
-    const assocFrom = Object.assign(Promise.resolve([]), { innerJoin: mock(() => assocInnerJoin) });
+    const assocFrom = Object.assign(Promise.resolve([]), {
+      innerJoin: mock(() => assocInnerJoin),
+    });
 
     const runsLimit = mock(() => Promise.resolve(prodOnly));
     const runsOrderBy = mock(() => ({ limit: runsLimit }));
@@ -180,7 +275,7 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
     });
 
     let selectCallCount = 0;
-    const mockDb = {
+    const mockDb = withTransactionMock({
       select: mock(() => {
         selectCallCount += 1;
         if (selectCallCount === 1) return { from: mock(() => assocFrom) };
@@ -196,10 +291,12 @@ describe("HealthCheckService - system rollup worst-wins across environments", ()
       update: mock(() => ({ set: mock(() => ({ where: mock(() => Promise.resolve()) })) })),
       delete: mock(() => ({ where: mock(() => Promise.resolve()) })),
       execute: mock(() => Promise.resolve()),
-    };
+    });
 
     const service = new HealthCheckService(mockDb as never, {} as never, {} as never);
     const result = await service.getSystemHealthStatus("system-1", "prod");
     expect(result.status).toBe("unhealthy");
+    expect(result.checkStatuses[0].sliceCount).toBe(1);
+    expect(result.checkStatuses[0].failingSliceCount).toBe(1);
   });
-});
+});