@checkstack/healthcheck-backend 1.17.0 → 1.18.0

package/src/slow-check-admission.ts

@@ -0,0 +1,101 @@
+/**
+ * The slow-check bulkhead + adaptive-timeout DECISION for one run of a
+ * `(configId, systemId, environmentId)` slice. Pure except for the lane
+ * admission side effect (a semaphore acquire), so it is directly unit-testable
+ * without the executor's DB/queue machinery. The executor supplies the slice's
+ * recent runs (read from durable `health_check_runs`) and acts on the verdict:
+ * a `defer` records nothing this tick, a `run` uses the (possibly shrunk)
+ * `effectiveTimeoutMs` and releases `laneKey` when set.
+ */
+import type { SlowCheckRuntime } from "./slow-check-config";
+import {
+  classifySlowCheck,
+  ENV_LESS_KEY,
+  type RecentRun,
+} from "./slow-check-classifier";
+import { adaptiveTimeout } from "./adaptive-timeout";
+
+export type SlowCheckDecision =
+  | {
+      kind: "run";
+      /** Timeout to probe with (shrunk toward the healthy baseline when suspect). */
+      effectiveTimeoutMs: number;
+      /** Set when a suspect run was admitted to the lane; release it after the run. */
+      laneKey?: string;
+    }
+  | {
+      kind: "defer";
+      /** `lane_full` (pod at capacity) or `in_flight` (prior run of this slice). */
+      reason: "lane_full" | "in_flight";
+    };
+
+/**
+ * Build the lane single-flight key for a slice. Keyed on `(config, system, env)`
+ * so distinct envs of one system get independent slots and a slice can never be
+ * in flight against itself.
+ */
+export function slowCheckLaneKey(props: {
+  configId: string;
+  systemId: string;
+  environmentId: string | null;
+}): string {
+  const { configId, systemId, environmentId } = props;
+  return `${configId}:${systemId}:${environmentId ?? ENV_LESS_KEY}`;
+}
+
+/**
+ * Decide whether to run this slice and with what timeout. A non-suspect slice
+ * always runs at the full timeout with no lane involvement. A suspect slice is
+ * admitted to the capped, pod-local lane (returning `laneKey` to release after
+ * the run) and probed with an adaptive timeout, OR deferred when the lane is
+ * full / a prior run of the same slice is still in flight.
+ */
+export function evaluateSlowCheckAdmission(props: {
+  runtime: SlowCheckRuntime;
+  recentRuns: RecentRun[];
+  configId: string;
+  systemId: string;
+  environmentId: string | null;
+  executionTimeoutMs: number;
+}): SlowCheckDecision {
+  const {
+    runtime,
+    recentRuns,
+    configId,
+    systemId,
+    environmentId,
+    executionTimeoutMs,
+  } = props;
+
+  const { perEnv } = classifySlowCheck({
+    runs: recentRuns,
+    params: {
+      ...runtime.classifierParams,
+      configuredTimeoutMs: executionTimeoutMs,
+    },
+  });
+  const classification = perEnv.get(environmentId ?? ENV_LESS_KEY);
+
+  if (!classification?.suspect) {
+    return { kind: "run", effectiveTimeoutMs: executionTimeoutMs };
+  }
+
+  const laneKey = slowCheckLaneKey({ configId, systemId, environmentId });
+  const admission = runtime.lane.tryAdmit(laneKey);
+  if (!admission.admitted) {
+    return { kind: "defer", reason: admission.reason };
+  }
+
+  return {
+    kind: "run",
+    laneKey,
+    effectiveTimeoutMs: adaptiveTimeout({
+      configuredMs: executionTimeoutMs,
+      healthyBaselineMs: classification.healthyBaselineMs,
+      isSuspect: true,
+      isRecoveryProbe: classification.isRecoveryProbe,
+      safetyFactor: runtime.safetyFactor,
+      absoluteFloorMs: runtime.absoluteFloorMs,
+    }),
+  };
+}

package/src/slow-check-classifier.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, test } from "bun:test";
+import {
+  classifySlowCheck,
+  ENV_LESS_KEY,
+  type RecentRun,
+} from "./slow-check-classifier";
+import type { HealthCheckStatus } from "@checkstack/healthcheck-common";
+
+const TIMEOUT = 30_000;
+const SLOW = TIMEOUT * 0.8; // 24_000
+
+// Build newest-first runs for one env.
+function runs(
+  entries: Array<{ status: HealthCheckStatus; latencyMs: number | null; env?: string | null }>,
+): RecentRun[] {
+  const now = Date.now();
+  return entries.map((e, i) => ({
+    status: e.status,
+    latencyMs: e.latencyMs,
+    environmentId: e.env === undefined ? null : e.env,
+    timestamp: new Date(now - i * 60_000),
+  }));
+}
+
+describe("classifySlowCheck", () => {
+  test("env-less: 3 slow timeouts => suspect", () => {
+    const r = runs([
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+    ]);
+    const c = classifySlowCheck({ runs: r, params: { configuredTimeoutMs: TIMEOUT } });
+    expect(c.perEnv.get(ENV_LESS_KEY)?.suspect).toBe(true);
+  });
+
+  test("fast connect-refused (low latency) is NOT suspect - it frees its slot", () => {
+    const r = runs([
+      { status: "unhealthy", latencyMs: 5 },
+      { status: "unhealthy", latencyMs: 8 },
+      { status: "unhealthy", latencyMs: 3 },
+    ]);
+    const c = classifySlowCheck({ runs: r, params: { configuredTimeoutMs: TIMEOUT } });
+    expect(c.perEnv.get(ENV_LESS_KEY)?.suspect).toBe(false);
+  });
+
+  test("one recent healthy run clears suspect (streak broken)", () => {
+    const r = runs([
+      { status: "healthy", latencyMs: 200 },
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+    ]);
+    const c = classifySlowCheck({ runs: r, params: { configuredTimeoutMs: TIMEOUT } });
+    expect(c.perEnv.get(ENV_LESS_KEY)?.suspect).toBe(false);
+  });
+
+  test("mixed envs: only the failing env is suspect; the healthy sibling is not", () => {
+    // The common multi-stage case: one env down, siblings healthy. The failing
+    // env is isolated per-env; the healthy sibling keeps running untouched.
+    const suspectEnv = runs([
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+    ]);
+    const healthyEnv = runs([
+      { status: "healthy", latencyMs: 150, env: "staging" },
+      { status: "healthy", latencyMs: 160, env: "staging" },
+    ]);
+    const c = classifySlowCheck({
+      runs: [...suspectEnv, ...healthyEnv],
+      params: { configuredTimeoutMs: TIMEOUT },
+    });
+    expect(c.perEnv.get("prod")?.suspect).toBe(true);
+    expect(c.perEnv.get("staging")?.suspect).toBe(false);
+  });
+
+  test("each env is classified independently", () => {
+    const prod = runs([
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+      { status: "unhealthy", latencyMs: SLOW, env: "prod" },
+    ]);
+    const staging = runs([
+      { status: "unhealthy", latencyMs: SLOW, env: "staging" },
+      { status: "unhealthy", latencyMs: SLOW, env: "staging" },
+      { status: "unhealthy", latencyMs: SLOW, env: "staging" },
+    ]);
+    const c = classifySlowCheck({
+      runs: [...prod, ...staging],
+      params: { configuredTimeoutMs: TIMEOUT },
+    });
+    expect(c.perEnv.get("prod")?.suspect).toBe(true);
+    expect(c.perEnv.get("staging")?.suspect).toBe(true);
+  });
+
+  test("healthyBaselineMs is p95 of HEALTHY runs only (excludes timed-out runs)", () => {
+    const r = runs([
+      { status: "unhealthy", latencyMs: SLOW }, // excluded from baseline
+      { status: "healthy", latencyMs: 100 },
+      { status: "healthy", latencyMs: 120 },
+      { status: "healthy", latencyMs: 110 },
+    ]);
+    const c = classifySlowCheck({ runs: r, params: { configuredTimeoutMs: TIMEOUT } });
+    const env = c.perEnv.get(ENV_LESS_KEY);
+    expect(env?.healthyBaselineMs).toBeDefined();
+    expect(env?.healthyBaselineMs).toBeLessThanOrEqual(120);
+    expect(env?.healthyBaselineMs).toBeGreaterThanOrEqual(100);
+  });
+
+  test("no healthy runs => healthyBaselineMs undefined (adaptive timeout won't shrink)", () => {
+    const r = runs([
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+      { status: "unhealthy", latencyMs: SLOW },
+    ]);
+    const c = classifySlowCheck({ runs: r, params: { configuredTimeoutMs: TIMEOUT } });
+    expect(c.perEnv.get(ENV_LESS_KEY)?.healthyBaselineMs).toBeUndefined();
+  });
+
+  test("recovery probe fires on the Nth consecutive suspect run", () => {
+    // 5 consecutive slow failures, recoveryProbeEvery=5 => 5 % 5 === 0 => probe.
+    const r = runs(
+      Array.from({ length: 5 }, () => ({
+        status: "unhealthy" as HealthCheckStatus,
+        latencyMs: SLOW,
+      })),
+    );
+    const c = classifySlowCheck({
+      runs: r,
+      params: { configuredTimeoutMs: TIMEOUT, recoveryProbeEvery: 5 },
+    });
+    expect(c.perEnv.get(ENV_LESS_KEY)?.isRecoveryProbe).toBe(true);
+  });
+
+  test("no probe when streak is not a multiple of the cadence", () => {
+    const r = runs(
+      Array.from({ length: 3 }, () => ({
+        status: "unhealthy" as HealthCheckStatus,
+        latencyMs: SLOW,
+      })),
+    );
+    const c = classifySlowCheck({
+      runs: r,
+      params: { configuredTimeoutMs: TIMEOUT, recoveryProbeEvery: 5, consecutiveFailures: 3 },
+    });
+    const env = c.perEnv.get(ENV_LESS_KEY);
+    expect(env?.suspect).toBe(true);
+    expect(env?.isRecoveryProbe).toBe(false); // 3 % 5 !== 0
+  });
+
+  test("empty history => no classified envs", () => {
+    const c = classifySlowCheck({ runs: [], params: { configuredTimeoutMs: TIMEOUT } });
+    expect(c.perEnv.size).toBe(0);
+  });
+});

package/src/slow-check-classifier.ts

@@ -0,0 +1,137 @@
+/**
+ * Classify a health check's recent runs to drive the slow-check bulkhead and the
+ * adaptive timeout. Pure and derived entirely from durable `health_check_runs`
+ * rows, so every pod computes the same classification (state-and-scale: no
+ * pod-local classification state).
+ *
+ * Granularities, matching the executor's execution model (one `(configId,
+ * systemId)` JOB holds one concurrency slot and runs its environments
+ * SEQUENTIALLY, each with its own timeout):
+ *
+ * Classification is PER ENV: each env's `suspect`, `healthyBaselineMs`, and
+ * `isRecoveryProbe` drive its OWN lane admission and adaptive timeout inside the
+ * job's execution loop. A failing env is admitted to the capped suspect lane (or
+ * skipped when the lane is full) and probed with a shrunk timeout, while a
+ * healthy sibling env in the SAME job runs normally at its full timeout in its
+ * own loop iteration. So one failing env is isolated without ever starving or
+ * dropping a healthy sibling env - and the bulkhead engages in the common
+ * mixed-stage case (only some envs down), not just when every env is down.
+ */
+import type { HealthCheckStatus } from "@checkstack/healthcheck-common";
+
+/** A recent run row, projected to the fields classification needs. */
+export interface RecentRun {
+  environmentId: string | null;
+  status: HealthCheckStatus;
+  latencyMs: number | null;
+  timestamp: Date;
+}
+
+export interface SlowCheckParams {
+  /** The check's configured execution timeout (ms) - the slot-hog latency ref. */
+  configuredTimeoutMs: number;
+  /** Consecutive slot-hog failures required to classify an env suspect. */
+  consecutiveFailures?: number;
+  /** Fraction of the timeout a failed run's latency must reach to count as a
+   *  slot-hog (vs a fast connect-refused that frees its slot instantly). */
+  slowFraction?: number;
+  /** Every Nth consecutive suspect run is a full-timeout recovery probe. */
+  recoveryProbeEvery?: number;
+}
+
+export interface EnvClassification {
+  suspect: boolean;
+  /** p95 latency (ms) of this env's recent HEALTHY runs; `undefined` if none. */
+  healthyBaselineMs: number | undefined;
+  /** This run should use the full configured timeout to re-measure recovery. */
+  isRecoveryProbe: boolean;
+}
+
+export interface SlowCheckClassification {
+  /** Per-env classification, keyed by `environmentId ?? ENV_LESS_KEY`. */
+  perEnv: Map<string, EnvClassification>;
+}
+
+export const DEFAULT_CONSECUTIVE_FAILURES = 3;
+export const DEFAULT_SLOW_FRACTION = 0.8;
+export const DEFAULT_RECOVERY_PROBE_EVERY = 5;
+/** Map key used for the env-less run (`environmentId === null`). */
+export const ENV_LESS_KEY = "_";
+
+/** p95 of a non-empty list (nearest-rank). */
+function p95(values: number[]): number {
+  const sorted = values.toSorted((a, b) => a - b);
+  const idx = Math.min(sorted.length - 1, Math.ceil(0.95 * sorted.length) - 1);
+  return sorted[Math.max(0, idx)]!;
+}
+
+/** True when a run held its slot ~for the timeout (a slow failure), not a fast
+ *  fail. Both connect-timeout and post-connect-timeout land here; a fast
+ *  connection-refused (low latency) does not. */
+function isSlotHogFailure(run: RecentRun, slowLatencyMs: number): boolean {
+  return run.status !== "healthy" && (run.latencyMs ?? 0) >= slowLatencyMs;
+}
+
+function classifyEnv(
+  runsNewestFirst: RecentRun[],
+  params: Required<SlowCheckParams>,
+): EnvClassification {
+  const { configuredTimeoutMs, consecutiveFailures, slowFraction, recoveryProbeEvery } =
+    params;
+  const slowLatencyMs = configuredTimeoutMs * slowFraction;
+
+  // Count leading consecutive slot-hog failures from the most recent run.
+  let leadingSlowFailures = 0;
+  for (const run of runsNewestFirst) {
+    if (isSlotHogFailure(run, slowLatencyMs)) leadingSlowFailures++;
+    else break;
+  }
+
+  const suspect = leadingSlowFailures >= consecutiveFailures;
+
+  const healthyLatencies = runsNewestFirst
+    .filter((r) => r.status === "healthy" && r.latencyMs !== null)
+    .map((r) => r.latencyMs!);
+  const healthyBaselineMs =
+    healthyLatencies.length > 0 ? p95(healthyLatencies) : undefined;
+
+  // Guardrail 3: every Nth consecutive suspect run is a full-timeout probe.
+  // Uses the failure streak BEFORE this run, so the streak lengths that trip a
+  // probe are recoveryProbeEvery, 2x, 3x, ...
+  const isRecoveryProbe =
+    suspect && leadingSlowFailures % recoveryProbeEvery === 0;
+
+  return { suspect, healthyBaselineMs, isRecoveryProbe };
+}
+
+/**
+ * Classify recent runs for one `(configId, systemId)` across all environments.
+ * `runs` must be newest-first (as `ORDER BY timestamp DESC` returns them).
+ */
+export function classifySlowCheck(props: {
+  runs: RecentRun[];
+  params: SlowCheckParams;
+}): SlowCheckClassification {
+  const params: Required<SlowCheckParams> = {
+    consecutiveFailures: DEFAULT_CONSECUTIVE_FAILURES,
+    slowFraction: DEFAULT_SLOW_FRACTION,
+    recoveryProbeEvery: DEFAULT_RECOVERY_PROBE_EVERY,
+    ...props.params,
+  };
+
+  // Bucket by env, preserving newest-first order within each bucket.
+  const byEnv = new Map<string, RecentRun[]>();
+  for (const run of props.runs) {
+    const key = run.environmentId ?? ENV_LESS_KEY;
+    const bucket = byEnv.get(key);
+    if (bucket) bucket.push(run);
+    else byEnv.set(key, [run]);
+  }
+
+  const perEnv = new Map<string, EnvClassification>();
+  for (const [key, envRuns] of byEnv) {
+    perEnv.set(key, classifyEnv(envRuns, params));
+  }
+
+  return { perEnv };
+}

package/src/slow-check-config.ts

@@ -0,0 +1,102 @@
+/**
+ * Runtime configuration for the slow-check bulkhead + adaptive timeout, resolved
+ * once at worker startup from environment variables. When disabled (the kill
+ * switch, or an invalid capacity), the executor skips the classification read
+ * and the whole feature is inert - health checks run exactly as before.
+ *
+ * The bulkhead lane is pod-local infrastructure (like the queue's own
+ * concurrency semaphore): total suspect concurrency across the cluster is
+ * `capacity x pods`, scaling the same way queue concurrency already does.
+ */
+import { SuspectLane } from "./suspect-lane";
+import {
+  DEFAULT_CONSECUTIVE_FAILURES,
+  DEFAULT_RECOVERY_PROBE_EVERY,
+  DEFAULT_SLOW_FRACTION,
+  type SlowCheckParams,
+} from "./slow-check-classifier";
+import {
+  DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
+  DEFAULT_TIMEOUT_SAFETY_FACTOR,
+} from "./adaptive-timeout";
+
+export interface SlowCheckRuntime {
+  /** Pod-local admission control for suspect (slot-hogging) env-runs. */
+  lane: SuspectLane;
+  /** How many recent runs (per config+system, across envs) to classify over. */
+  recentRunsLimit: number;
+  /** Classifier tuning (consecutive-failure count, slow fraction, probe cadence). */
+  classifierParams: Omit<SlowCheckParams, "configuredTimeoutMs">;
+  /** Adaptive-timeout multiplier on the healthy-latency baseline. */
+  safetyFactor: number;
+  /** Adaptive-timeout absolute lower bound (ms). */
+  absoluteFloorMs: number;
+}
+
+export const DEFAULT_SLOW_LANE_CAPACITY = 3;
+export const DEFAULT_RECENT_RUNS_LIMIT = 20;
+
+function numberFrom(raw: string | undefined, fallback: number): number {
+  if (raw === undefined) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function isDisabled(raw: string | undefined): boolean {
+  return raw === "0" || raw?.toLowerCase() === "false";
+}
+
+/**
+ * Resolve the slow-check runtime from env vars. Returns `null` when the feature
+ * is disabled (kill switch, or a non-positive capacity), which the executor
+ * treats as "run exactly as before".
+ */
+export function resolveSlowCheckRuntime(
+  env: Record<string, string | undefined>,
+): SlowCheckRuntime | null {
+  if (isDisabled(env.CHECKSTACK_HEALTHCHECK_SLOW_LANE_ENABLED)) return null;
+
+  const capacity = Math.floor(
+    numberFrom(env.CHECKSTACK_HEALTHCHECK_SLOW_LANE_CAPACITY, DEFAULT_SLOW_LANE_CAPACITY),
+  );
+  if (!Number.isInteger(capacity) || capacity < 1) return null;
+
+  return {
+    lane: new SuspectLane(capacity),
+    recentRunsLimit: Math.max(
+      1,
+      Math.floor(
+        numberFrom(env.CHECKSTACK_HEALTHCHECK_SLOW_RECENT_RUNS, DEFAULT_RECENT_RUNS_LIMIT),
+      ),
+    ),
+    classifierParams: {
+      consecutiveFailures: Math.max(
+        1,
+        Math.floor(
+          numberFrom(
+            env.CHECKSTACK_HEALTHCHECK_SLOW_CONSECUTIVE_FAILURES,
+            DEFAULT_CONSECUTIVE_FAILURES,
+          ),
+        ),
+      ),
+      slowFraction: numberFrom(env.CHECKSTACK_HEALTHCHECK_SLOW_FRACTION, DEFAULT_SLOW_FRACTION),
+      recoveryProbeEvery: Math.max(
+        1,
+        Math.floor(
+          numberFrom(
+            env.CHECKSTACK_HEALTHCHECK_SLOW_RECOVERY_PROBE_EVERY,
+            DEFAULT_RECOVERY_PROBE_EVERY,
+          ),
+        ),
+      ),
+    },
+    safetyFactor: numberFrom(
+      env.CHECKSTACK_HEALTHCHECK_SLOW_SAFETY_FACTOR,
+      DEFAULT_TIMEOUT_SAFETY_FACTOR,
+    ),
+    absoluteFloorMs: numberFrom(
+      env.CHECKSTACK_HEALTHCHECK_SLOW_FLOOR_MS,
+      DEFAULT_TIMEOUT_ABSOLUTE_FLOOR_MS,
+    ),
+  };
+}

package/src/suspect-lane.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "bun:test";
+import { SuspectLane } from "./suspect-lane";
+
+describe("SuspectLane", () => {
+  test("admits up to capacity, then denies with lane_full", () => {
+    const lane = new SuspectLane(2);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("b")).toEqual({ admitted: true });
+    expect(lane.active).toBe(2);
+    expect(lane.tryAdmit("c")).toEqual({ admitted: false, reason: "lane_full" });
+  });
+
+  test("single-flight: a second admit of the same key is denied with in_flight", () => {
+    const lane = new SuspectLane(4);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    expect(lane.active).toBe(1); // the duplicate did not consume a slot
+  });
+
+  test("in_flight is checked before capacity", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    // lane is full AND k is in-flight; in_flight wins (more specific reason).
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+  });
+
+  test("release frees the slot and clears single-flight", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    lane.release("k");
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true }); // re-admittable
+  });
+
+  test("release is idempotent for unknown/duplicate keys", () => {
+    const lane = new SuspectLane(2);
+    lane.tryAdmit("a");
+    lane.release("a");
+    lane.release("a"); // no-op
+    lane.release("never-admitted"); // no-op
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+  });
+
+  test("rejects a non-positive capacity", () => {
+    expect(() => new SuspectLane(0)).toThrow();
+    expect(() => new SuspectLane(-1)).toThrow();
+  });
+});

package/src/suspect-lane.ts

@@ -0,0 +1,61 @@
+/**
+ * Pod-local bulkhead for slot-hogging ("suspect") health-check jobs.
+ *
+ * A correlated outage can turn hundreds of checks into slot-hoggers at once. The
+ * work-conserving bulkhead caps how many suspect jobs may HOLD a worker slot
+ * concurrently on this pod, so healthy checks keep draining. It is deliberately
+ * NON-BLOCKING: a suspect job that cannot be admitted is skipped for this tick
+ * (freeing its worker slot immediately) rather than queued behind the lane -
+ * queueing would just move the starvation back into the shared worker pool.
+ *
+ * Two admission gates, combined:
+ *   - single-flight: at most ONE in-flight run per `(configId, systemId)` job
+ *     key, so a recurring fire that arrives while the previous slow run is still
+ *     executing is dropped instead of stacking up (no buildup).
+ *   - capacity: at most `capacity` suspect jobs holding a slot at once.
+ *
+ * This is pod-local infrastructure, exactly like the queue's own concurrency
+ * semaphore: total suspect concurrency across the cluster is `capacity x pods`,
+ * scaling the same way queue concurrency already does. The CLASSIFICATION that
+ * decides which jobs are suspect derives from durable run history (globally
+ * consistent); only this admission bookkeeping is per-pod.
+ */
+
+export type AdmissionDenial = "in_flight" | "lane_full";
+
+export type AdmissionResult =
+  | { admitted: true }
+  | { admitted: false; reason: AdmissionDenial };
+
+export class SuspectLane {
+  private held = 0;
+  private readonly inFlight = new Set<string>();
+
+  constructor(private readonly capacity: number) {
+    if (!Number.isInteger(capacity) || capacity < 1) {
+      throw new Error(`SuspectLane capacity must be a positive integer, got ${capacity}`);
+    }
+  }
+
+  /**
+   * Try to admit a suspect job. On `admitted: true` the caller MUST call
+   * `release(key)` exactly once (in a `finally`) when the run completes.
+   */
+  tryAdmit(key: string): AdmissionResult {
+    if (this.inFlight.has(key)) return { admitted: false, reason: "in_flight" };
+    if (this.held >= this.capacity) return { admitted: false, reason: "lane_full" };
+    this.held++;
+    this.inFlight.add(key);
+    return { admitted: true };
+  }
+
+  /** Release a previously-admitted key. Idempotent for an unknown key. */
+  release(key: string): void {
+    if (this.inFlight.delete(key)) this.held--;
+  }
+
+  /** Number of suspect jobs currently holding a slot (for metrics/tests). */
+  get active(): number {
+    return this.held;
+  }
+}