@checkstack/healthcheck-backend 1.17.0 → 1.18.0

package/src/service.ts

@@ -43,6 +43,7 @@ import {
   isNull,
   isNotNull,
   inArray,
+  max,
 } from "drizzle-orm";
 import { ORPCError } from "@orpc/server";
 import { evaluateHealthStatus } from "./state-evaluator";
@@ -51,7 +52,9 @@ import { parseHealthEntityId } from "./health-entity-id";
 import { stateThresholds } from "./state-thresholds-migrations";
 import type { MaintenanceApi } from "@checkstack/maintenance-common";
 import type { Logger } from "@checkstack/backend-api";
+import { resolveEffectiveEnvironments } from "./effective-environments";
 import { incrementHourlyAggregate } from "./realtime-aggregation";
+import { withScopedTransaction } from "@checkstack/backend-api";
 import type {
   HealthCheckRegistry,
   SafeDatabase,
@@ -107,6 +110,10 @@ interface SystemCheckStatus {
   status: HealthCheckStatus;
   runsConsidered: number;
   lastRunAt?: Date;
+  /** Environment slices this check currently fans out to (>= 1). */
+  sliceCount: number;
+  /** How many of {@link sliceCount} slices are currently non-healthy. */
+  failingSliceCount: number;
 }
 
 interface SystemHealthStatusResponse {
@@ -260,6 +267,53 @@ export class HealthCheckService {
     return config ? this.mapConfig(config) : undefined;
   }
 
+  /**
+   * Resolve the per-environment slices a (system, config) assignment should
+   * enqueue for a ONE-OFF run (the `run_now` automation). Returns the list of
+   * environment ids to run, or `[null]` (a single env-less run) when the
+   * assignment has no effective environments. Mirrors the executor's fan-out
+   * resolution so a manual run covers exactly the same slices the recurring
+   * schedule does. Fail-open: a catalog resolution failure collapses to a
+   * single env-less run rather than enqueuing nothing.
+   */
+  async resolveEnqueueEnvironmentIds(props: {
+    systemId: string;
+    configurationId: string;
+    catalogClient: CatalogClient;
+    logger: Logger;
+  }): Promise<(string | null)[]> {
+    const { systemId, configurationId, catalogClient, logger } = props;
+    const [assignment] = await this.db
+      .select({ environmentIds: systemHealthChecks.environmentIds })
+      .from(systemHealthChecks)
+      .where(
+        and(
+          eq(systemHealthChecks.systemId, systemId),
+          eq(systemHealthChecks.configurationId, configurationId),
+        ),
+      );
+
+    let membership: Awaited<
+      ReturnType<CatalogClient["resolveSystemEnvironments"]>
+    > = [];
+    try {
+      membership = await catalogClient.resolveSystemEnvironments({ systemId });
+    } catch (error) {
+      logger.warn(
+        `run_now: could not resolve environments for system ${systemId}`,
+        error,
+      );
+    }
+
+    const effectiveEnvs = resolveEffectiveEnvironments({
+      environmentIds: assignment?.environmentIds,
+      membership,
+    });
+    return effectiveEnvs.length > 0
+      ? effectiveEnvs.map((env) => env.id)
+      : [null];
+  }
+
   /**
    * Redact a configuration for a UI/AI read: every `x-secret` field is
    * removed from the strategy config and each collector config. Stored
@@ -1030,159 +1084,179 @@ export class HealthCheckService {
     systemId: string,
     environmentId?: string | null,
   ): Promise<SystemHealthStatusResponse> {
-    // Get all associations for this system with their thresholds and config names
-    const associations = await this.db
-      .select({
-        configurationId: systemHealthChecks.configurationId,
-        stateThresholds: systemHealthChecks.stateThresholds,
-        configName: healthCheckConfigurations.name,
-        enabled: systemHealthChecks.enabled,
-      })
-      .from(systemHealthChecks)
-      .innerJoin(
-        healthCheckConfigurations,
-        eq(systemHealthChecks.configurationId, healthCheckConfigurations.id),
-      )
-      .where(
-        and(
-          eq(systemHealthChecks.systemId, systemId),
-          eq(systemHealthChecks.enabled, true),
-          // A paused configuration contributes no signal to the system's
-          // health: its execution is skipped (see queue-executor pause gate)
-          // and its historical runs MUST NOT keep the aggregate degraded
-          // while it is paused. Excluding it here makes the rollup reflect
-          // only the actively-running checks, so pausing the sole failing
-          // check clears the system's status and downstream SLO downtime.
-          eq(healthCheckConfigurations.paused, false),
-        ),
-      );
-
-    if (associations.length === 0) {
-      // No health checks configured - default healthy
-      return {
-        status: "healthy",
-        evaluatedAt: new Date(),
-        checkStatuses: [],
-      };
-    }
-
-    // For each association, get recent runs and evaluate status
-    const checkStatuses: SystemCheckStatus[] = [];
-    const maxWindowSize = 100; // Max configurable window size
-
-    // Environment filter for the per-check run window. `undefined` (rollup)
-    // adds no predicate; `null` filters to the env-less slice; a string
-    // filters to that environment. The lookup index leads with
-    // (system_id, environment_id, …) so the env-scoped query is index-efficient.
-    //
-    // For the rollup, we deliberately do NOT apply a single envFilter to one
-    // flat run list — see the per-association branch below for why.
-    const envFilter =
-      environmentId === undefined
-        ? undefined
-        : environmentId === null
-          ? isNull(healthCheckRuns.environmentId)
-          : eq(healthCheckRuns.environmentId, environmentId);
-
-    for (const assoc of associations) {
-      // Extract and migrate thresholds from versioned config
-      let thresholds: StateThresholds | undefined;
-      if (assoc.stateThresholds) {
-        thresholds = await stateThresholds.parse(assoc.stateThresholds);
-      }
+    // §perf: batch the 1 (associations) + N (per-check run window) reads
+    // into ONE scoped transaction so the whole read fan-out pays a single
+    // BEGIN/SET LOCAL/COMMIT and holds one connection, instead of 1+N
+    // standalone scoped queries each checking a connection out. Pure
+    // evaluation runs inside too (it issues no DB). See withScopedTransaction.
+    const checkStatuses = await withScopedTransaction(this.db, async (tx) => {
+      // Get all associations for this system with their thresholds and config names
+      const associations = await tx
+        .select({
+          configurationId: systemHealthChecks.configurationId,
+          stateThresholds: systemHealthChecks.stateThresholds,
+          configName: healthCheckConfigurations.name,
+          enabled: systemHealthChecks.enabled,
+        })
+        .from(systemHealthChecks)
+        .innerJoin(
+          healthCheckConfigurations,
+          eq(systemHealthChecks.configurationId, healthCheckConfigurations.id),
+        )
+        .where(
+          and(
+            eq(systemHealthChecks.systemId, systemId),
+            eq(systemHealthChecks.enabled, true),
+            // A paused configuration contributes no signal to the system's
+            // health: its execution is skipped (see queue-executor pause gate)
+            // and its historical runs MUST NOT keep the aggregate degraded
+            // while it is paused. Excluding it here makes the rollup reflect
+            // only the actively-running checks, so pausing the sole failing
+            // check clears the system's status and downstream SLO downtime.
+            eq(healthCheckConfigurations.paused, false),
+          ),
+        );
 
-      let status: HealthCheckStatus;
-      let runsConsidered: number;
-      let lastRunAt: Date | undefined;
-
-      if (environmentId === undefined) {
-        // System rollup: evaluate the threshold window PER ENVIRONMENT within
-        // the association, then take worst-wins ACROSS envs. Flattening every
-        // env's runs into one list feeds interleaved statuses to
-        // `evaluateConsecutive` (the default mode): the streak breaks on the
-        // first interleaving env, so the evaluator collapses to whatever
-        // single env's status the most recent run landed on. That masks any
-        // permanently-failing sibling env in the default mode ("the healthy
-        // env wins"), and flaps healthy↔degraded whenever env insertion
-        // order drifts across ticks (see the regression test
-        // `rollup — worst-wins across environments within an association`).
-        // Per-env evaluation makes the rollup worst-wins stable regardless of
-        // insertion order or multi-pod racing.
-        const runs = await this.db
-          .select({
-            status: healthCheckRuns.status,
-            timestamp: healthCheckRuns.timestamp,
-            environmentId: healthCheckRuns.environmentId,
-          })
-          .from(healthCheckRuns)
-          .where(
-            and(
-              eq(healthCheckRuns.systemId, systemId),
-              eq(healthCheckRuns.configurationId, assoc.configurationId),
-            ),
-          )
-          .orderBy(desc(healthCheckRuns.timestamp))
-          .limit(maxWindowSize);
-
-        // Group by environmentId. `null` is its own group (the env-less slice
-        // of an assignment that has opted out, plus any pre-3b env-less runs).
-        const byEnv = new Map<string | null, { status: HealthCheckStatus; timestamp: Date }[]>();
-        for (const r of runs) {
-          const key = r.environmentId ?? null;
-          const bucket = byEnv.get(key);
-          if (bucket) {
-            bucket.push(r);
-          } else {
-            byEnv.set(key, [r]);
-          }
+      if (associations.length === 0) return [];
+
+      // For each association, get recent runs and evaluate status
+      const out: SystemCheckStatus[] = [];
+      const maxWindowSize = 100; // Max configurable window size
+
+      // Environment filter for the per-check run window. `undefined` (rollup)
+      // adds no predicate; `null` filters to the env-less slice; a string
+      // filters to that environment. The lookup index leads with
+      // (system_id, environment_id, …) so the env-scoped query is index-efficient.
+      //
+      // For the rollup, we deliberately do NOT apply a single envFilter to one
+      // flat run list — see the per-association branch below for why.
+      const envFilter =
+        environmentId === undefined
+          ? undefined
+          : environmentId === null
+            ? isNull(healthCheckRuns.environmentId)
+            : eq(healthCheckRuns.environmentId, environmentId);
+
+      for (const assoc of associations) {
+        // Extract and migrate thresholds from versioned config
+        let thresholds: StateThresholds | undefined;
+        if (assoc.stateThresholds) {
+          thresholds = await stateThresholds.parse(assoc.stateThresholds);
         }
 
-        status = "healthy";
-        runsConsidered = runs.length;
-        lastRunAt = runs[0]?.timestamp;
-        for (const envRuns of byEnv.values()) {
-          const envStatus = evaluateHealthStatus({ runs: envRuns, thresholds });
-          if (envStatus === "unhealthy") {
-            status = "unhealthy";
-            break; // worst: stop
+        let status: HealthCheckStatus;
+        let runsConsidered: number;
+        let lastRunAt: Date | undefined;
+        // Fan-out accounting for the honest "X of Y checks failing" denominator:
+        // how many environment slices this check currently spans, and how many
+        // are non-healthy. A non-fanned (single-env / env-less) check is one
+        // slice. Populated in both branches so the DTO field is always present.
+        let sliceCount = 1;
+        let failingSliceCount = 0;
+
+        if (environmentId === undefined) {
+          // System rollup: evaluate the threshold window PER ENVIRONMENT within
+          // the association, then take worst-wins ACROSS envs. Flattening every
+          // env's runs into one list feeds interleaved statuses to
+          // `evaluateConsecutive` (the default mode): the streak breaks on the
+          // first interleaving env, so the evaluator collapses to whatever
+          // single env's status the most recent run landed on. That masks any
+          // permanently-failing sibling env in the default mode ("the healthy
+          // env wins"), and flaps healthy↔degraded whenever env insertion
+          // order drifts across ticks (see the regression test
+          // `rollup — worst-wins across environments within an association`).
+          // Per-env evaluation makes the rollup worst-wins stable regardless of
+          // insertion order or multi-pod racing.
+          const runs = await tx
+            .select({
+              status: healthCheckRuns.status,
+              timestamp: healthCheckRuns.timestamp,
+              environmentId: healthCheckRuns.environmentId,
+            })
+            .from(healthCheckRuns)
+            .where(
+              and(
+                eq(healthCheckRuns.systemId, systemId),
+                eq(healthCheckRuns.configurationId, assoc.configurationId),
+              ),
+            )
+            .orderBy(desc(healthCheckRuns.timestamp))
+            .limit(maxWindowSize);
+
+          // Group by environmentId. `null` is its own group (the env-less slice
+          // of an assignment that has opted out, plus any pre-3b env-less runs).
+          const byEnv = new Map<string | null, { status: HealthCheckStatus; timestamp: Date }[]>();
+          for (const r of runs) {
+            const key = r.environmentId ?? null;
+            const bucket = byEnv.get(key);
+            if (bucket) {
+              bucket.push(r);
+            } else {
+              byEnv.set(key, [r]);
+            }
           }
-          if (envStatus === "degraded" && status === "healthy") {
-            status = "degraded";
+
+          status = "healthy";
+          runsConsidered = runs.length;
+          lastRunAt = runs[0]?.timestamp;
+          // Each env group is a slice. A check that has runs against N envs
+          // currently fans out to N; before it has ever run it is still one
+          // logical slice (byEnv empty => keep the default 1).
+          sliceCount = Math.max(byEnv.size, 1);
+          failingSliceCount = 0;
+          for (const envRuns of byEnv.values()) {
+            const envStatus = evaluateHealthStatus({ runs: envRuns, thresholds });
+            // Count EVERY failing slice (don't break early): the failing count
+            // feeds the dashboard numerator, so all non-healthy envs must tally.
+            if (envStatus !== "healthy") {
+              failingSliceCount++;
+            }
+            if (envStatus === "unhealthy") {
+              status = "unhealthy";
+            } else if (envStatus === "degraded" && status === "healthy") {
+              status = "degraded";
+            }
           }
+        } else {
+          // Per-env (string) or env-less (null) slice: that slice's flat run
+          // window is monotonic per-env, so the threshold evaluator sees no
+          // interleaving — the consecutive streak is well-defined.
+          const runs = await tx
+            .select({
+              status: healthCheckRuns.status,
+              timestamp: healthCheckRuns.timestamp,
+            })
+            .from(healthCheckRuns)
+            .where(
+              and(
+                eq(healthCheckRuns.systemId, systemId),
+                eq(healthCheckRuns.configurationId, assoc.configurationId),
+                ...(envFilter ? [envFilter] : []),
+              ),
+            )
+            .orderBy(desc(healthCheckRuns.timestamp))
+            .limit(maxWindowSize);
+
+          status = evaluateHealthStatus({ runs, thresholds });
+          runsConsidered = runs.length;
+          lastRunAt = runs[0]?.timestamp;
+          // Single-slice evaluation: this env either counts as failing or not.
+          sliceCount = 1;
+          failingSliceCount = status === "healthy" ? 0 : 1;
         }
-      } else {
-        // Per-env (string) or env-less (null) slice: that slice's flat run
-        // window is monotonic per-env, so the threshold evaluator sees no
-        // interleaving — the consecutive streak is well-defined.
-        const runs = await this.db
-          .select({
-            status: healthCheckRuns.status,
-            timestamp: healthCheckRuns.timestamp,
-          })
-          .from(healthCheckRuns)
-          .where(
-            and(
-              eq(healthCheckRuns.systemId, systemId),
-              eq(healthCheckRuns.configurationId, assoc.configurationId),
-              ...(envFilter ? [envFilter] : []),
-            ),
-          )
-          .orderBy(desc(healthCheckRuns.timestamp))
-          .limit(maxWindowSize);
 
-        status = evaluateHealthStatus({ runs, thresholds });
-        runsConsidered = runs.length;
-        lastRunAt = runs[0]?.timestamp;
+        out.push({
+          configurationId: assoc.configurationId,
+          configurationName: assoc.configName,
+          status,
+          runsConsidered,
+          lastRunAt,
+          sliceCount,
+          failingSliceCount,
+        });
       }
-
-      checkStatuses.push({
-        configurationId: assoc.configurationId,
-        configurationName: assoc.configName,
-        status,
-        runsConsidered,
-        lastRunAt,
-      });
-    }
+      return out;
+    });
 
     // Aggregate status: worst status wins (unhealthy > degraded > healthy)
     let aggregateStatus: HealthCheckStatus = "healthy";
@@ -1488,6 +1562,39 @@ export class HealthCheckService {
         thresholds = await stateThresholds.parse(assoc.stateThresholds);
       }
 
+      // Most recent HEALTHY run per environment, computed OUTSIDE the bounded
+      // sparkline window so "last successful run" stays correct even when a
+      // check has been failing for far longer than the last 25 runs. One
+      // grouped aggregate query per check (env-less = the `null` group). The
+      // (system_id, configuration_id, environment_id, timestamp) index makes
+      // this a cheap max-per-group scan.
+      const lastHealthyRows = await this.db
+        .select({
+          environmentId: healthCheckRuns.environmentId,
+          lastSuccessAt: max(healthCheckRuns.timestamp),
+        })
+        .from(healthCheckRuns)
+        .where(
+          and(
+            eq(healthCheckRuns.systemId, systemId),
+            eq(healthCheckRuns.configurationId, assoc.configurationId),
+            eq(healthCheckRuns.status, "healthy"),
+          ),
+        )
+        .groupBy(healthCheckRuns.environmentId);
+      const lastHealthyByEnv = new Map<string | null, Date>();
+      let checkLastSuccessfulRunAt: Date | undefined;
+      for (const row of lastHealthyRows) {
+        if (!row.lastSuccessAt) continue;
+        lastHealthyByEnv.set(row.environmentId ?? null, row.lastSuccessAt);
+        if (
+          !checkLastSuccessfulRunAt ||
+          row.lastSuccessAt > checkLastSuccessfulRunAt
+        ) {
+          checkLastSuccessfulRunAt = row.lastSuccessAt;
+        }
+      }
+
       // Group the fetched runs by environmentId (null = env-less slice). We
       // query each env's slice separately below to evaluate it on its own
       // monotonic run window and worst-wins across envs — this is the same
@@ -1497,6 +1604,7 @@ export class HealthCheckService {
       const perEnvironment: {
         environmentId: string | null;
         status: HealthCheckStatus;
+        lastSuccessfulRunAt?: Date;
         recentRuns: { id: string; status: HealthCheckStatus; timestamp: Date }[];
       }[] = [];
 
@@ -1552,6 +1660,7 @@ export class HealthCheckService {
         perEnvironment.push({
           environmentId: envId,
           status: envStatus,
+          lastSuccessfulRunAt: lastHealthyByEnv.get(envId),
           recentRuns: envRuns.toReversed().map((r) => ({
             id: r.id,
             status: r.status,
@@ -1581,6 +1690,7 @@ export class HealthCheckService {
         paused: assoc.paused,
         status,
         stateThresholds: thresholds,
+        lastSuccessfulRunAt: checkLastSuccessfulRunAt,
         recentRuns: chronologicalRuns.map((r) => ({
           id: r.id,
           status: r.status,

package/src/slow-check-admission.test.ts

@@ -0,0 +1,184 @@
+import { describe, it, expect } from "bun:test";
+import {
+  evaluateSlowCheckAdmission,
+  slowCheckLaneKey,
+} from "./slow-check-admission";
+import { SuspectLane } from "./suspect-lane";
+import type { SlowCheckRuntime } from "./slow-check-config";
+import type { RecentRun } from "./slow-check-classifier";
+
+function runtime(overrides: Partial<SlowCheckRuntime> = {}): SlowCheckRuntime {
+  return {
+    lane: new SuspectLane(1),
+    recentRunsLimit: 20,
+    classifierParams: {
+      consecutiveFailures: 3,
+      slowFraction: 0.8,
+      recoveryProbeEvery: 5,
+    },
+    safetyFactor: 1.5,
+    absoluteFloorMs: 1000,
+    ...overrides,
+  };
+}
+
+const TIMEOUT = 10_000;
+
+/** A slow transport failure (held its slot ~the full timeout). */
+function slowFail(): RecentRun {
+  return {
+    environmentId: null,
+    status: "unhealthy",
+    latencyMs: TIMEOUT, // >= slowFraction * timeout
+    timestamp: new Date(),
+  };
+}
+
+/** A healthy run with a given latency. */
+function healthy(latencyMs: number): RecentRun {
+  return { environmentId: null, status: "healthy", latencyMs, timestamp: new Date() };
+}
+
+describe("slowCheckLaneKey", () => {
+  it("keys on (config, system, env) with ENV_LESS_KEY for null", () => {
+    expect(
+      slowCheckLaneKey({ configId: "c", systemId: "s", environmentId: null }),
+    ).toBe("c:s:_");
+    expect(
+      slowCheckLaneKey({ configId: "c", systemId: "s", environmentId: "prod" }),
+    ).toBe("c:s:prod");
+  });
+});
+
+describe("evaluateSlowCheckAdmission", () => {
+  it("runs a NON-suspect slice at the full timeout with no lane involvement", () => {
+    const rt = runtime();
+    const decision = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: [healthy(50), healthy(60)],
+      configId: "c",
+      systemId: "s",
+      environmentId: null,
+      executionTimeoutMs: TIMEOUT,
+    });
+    expect(decision).toEqual({ kind: "run", effectiveTimeoutMs: TIMEOUT });
+    // No slot was taken.
+    expect(rt.lane.active).toBe(0);
+  });
+
+  it("admits a suspect slice and shrinks the timeout toward its healthy baseline", () => {
+    const rt = runtime();
+    const decision = evaluateSlowCheckAdmission({
+      runtime: rt,
+      // 3 leading slow failures ⇒ suspect; one healthy sample gives a baseline.
+      recentRuns: [slowFail(), slowFail(), slowFail(), healthy(200)],
+      configId: "c",
+      systemId: "s",
+      environmentId: null,
+      executionTimeoutMs: TIMEOUT,
+    });
+    expect(decision.kind).toBe("run");
+    if (decision.kind !== "run") return;
+    // adaptiveTimeout: max(floor 1000, 200 * 1.5) = 1000, and < configured.
+    expect(decision.effectiveTimeoutMs).toBe(1000);
+    expect(decision.laneKey).toBe("c:s:_");
+    expect(rt.lane.active).toBe(1);
+  });
+
+  it("keeps the full timeout on a recovery-probe suspect run", () => {
+    const rt = runtime();
+    // 5 leading slow failures ⇒ suspect AND a recovery probe (every 5th).
+    const decision = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: [
+        slowFail(),
+        slowFail(),
+        slowFail(),
+        slowFail(),
+        slowFail(),
+        healthy(200),
+      ],
+      configId: "c",
+      systemId: "s",
+      environmentId: null,
+      executionTimeoutMs: TIMEOUT,
+    });
+    expect(decision.kind).toBe("run");
+    if (decision.kind !== "run") return;
+    // Recovery probe re-measures at the FULL configured timeout.
+    expect(decision.effectiveTimeoutMs).toBe(TIMEOUT);
+    expect(decision.laneKey).toBe("c:s:_");
+  });
+
+  it("DEFERS a suspect slice when the lane is full (lane_full)", () => {
+    const rt = runtime({ lane: new SuspectLane(1) });
+    // Fill the single slot with a different slice.
+    rt.lane.tryAdmit("other:slice:_");
+
+    const decision = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: [slowFail(), slowFail(), slowFail()],
+      configId: "c",
+      systemId: "s",
+      environmentId: null,
+      executionTimeoutMs: TIMEOUT,
+    });
+    expect(decision).toEqual({ kind: "defer", reason: "lane_full" });
+  });
+
+  it("DEFERS a suspect slice already in flight (in_flight, single-flight)", () => {
+    const rt = runtime({ lane: new SuspectLane(4) });
+    // Same slice already holds a slot (a prior tick still running).
+    rt.lane.tryAdmit("c:s:_");
+
+    const decision = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: [slowFail(), slowFail(), slowFail()],
+      configId: "c",
+      systemId: "s",
+      environmentId: null,
+      executionTimeoutMs: TIMEOUT,
+    });
+    expect(decision).toEqual({ kind: "defer", reason: "in_flight" });
+  });
+
+  it("classifies per-env: one env suspect, a sibling env healthy in the same runtime", () => {
+    const rt = runtime({ lane: new SuspectLane(2) });
+    const prodFail = (): RecentRun => ({
+      environmentId: "prod",
+      status: "unhealthy",
+      latencyMs: TIMEOUT,
+      timestamp: new Date(),
+    });
+    const runs: RecentRun[] = [
+      prodFail(),
+      prodFail(),
+      prodFail(),
+      { environmentId: "staging", status: "healthy", latencyMs: 40, timestamp: new Date() },
+    ];
+
+    const prod = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: runs,
+      configId: "c",
+      systemId: "s",
+      environmentId: "prod",
+      executionTimeoutMs: TIMEOUT,
+    });
+    const staging = evaluateSlowCheckAdmission({
+      runtime: rt,
+      recentRuns: runs,
+      configId: "c",
+      systemId: "s",
+      environmentId: "staging",
+      executionTimeoutMs: TIMEOUT,
+    });
+
+    // prod is suspect (admitted, shrunk); staging runs at full timeout, no slot.
+    expect(prod.kind).toBe("run");
+    if (prod.kind === "run") expect(prod.laneKey).toBe("c:s:prod");
+    expect(staging).toEqual({ kind: "run", effectiveTimeoutMs: TIMEOUT });
+    // Only the suspect env took a slot.
+    expect(rt.lane.active).toBe(1);
+  });
+});