@checkstack/healthcheck-backend 1.17.0 → 1.19.0

package/src/suspect-lane.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "bun:test";
+import { SuspectLane } from "./suspect-lane";
+
+describe("SuspectLane", () => {
+  test("admits up to capacity, then denies with lane_full", () => {
+    const lane = new SuspectLane(2);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("b")).toEqual({ admitted: true });
+    expect(lane.active).toBe(2);
+    expect(lane.tryAdmit("c")).toEqual({ admitted: false, reason: "lane_full" });
+  });
+
+  test("single-flight: a second admit of the same key is denied with in_flight", () => {
+    const lane = new SuspectLane(4);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    expect(lane.active).toBe(1); // the duplicate did not consume a slot
+  });
+
+  test("in_flight is checked before capacity", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    // lane is full AND k is in-flight; in_flight wins (more specific reason).
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+  });
+
+  test("release frees the slot and clears single-flight", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    lane.release("k");
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true }); // re-admittable
+  });
+
+  test("release is idempotent for unknown/duplicate keys", () => {
+    const lane = new SuspectLane(2);
+    lane.tryAdmit("a");
+    lane.release("a");
+    lane.release("a"); // no-op
+    lane.release("never-admitted"); // no-op
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+  });
+
+  test("rejects a non-positive capacity", () => {
+    expect(() => new SuspectLane(0)).toThrow();
+    expect(() => new SuspectLane(-1)).toThrow();
+  });
+});

package/src/suspect-lane.ts

@@ -0,0 +1,61 @@
+/**
+ * Pod-local bulkhead for slot-hogging ("suspect") health-check jobs.
+ *
+ * A correlated outage can turn hundreds of checks into slot-hoggers at once. The
+ * work-conserving bulkhead caps how many suspect jobs may HOLD a worker slot
+ * concurrently on this pod, so healthy checks keep draining. It is deliberately
+ * NON-BLOCKING: a suspect job that cannot be admitted is skipped for this tick
+ * (freeing its worker slot immediately) rather than queued behind the lane -
+ * queueing would just move the starvation back into the shared worker pool.
+ *
+ * Two admission gates, combined:
+ *   - single-flight: at most ONE in-flight run per `(configId, systemId)` job
+ *     key, so a recurring fire that arrives while the previous slow run is still
+ *     executing is dropped instead of stacking up (no buildup).
+ *   - capacity: at most `capacity` suspect jobs holding a slot at once.
+ *
+ * This is pod-local infrastructure, exactly like the queue's own concurrency
+ * semaphore: total suspect concurrency across the cluster is `capacity x pods`,
+ * scaling the same way queue concurrency already does. The CLASSIFICATION that
+ * decides which jobs are suspect derives from durable run history (globally
+ * consistent); only this admission bookkeeping is per-pod.
+ */
+
+export type AdmissionDenial = "in_flight" | "lane_full";
+
+export type AdmissionResult =
+  | { admitted: true }
+  | { admitted: false; reason: AdmissionDenial };
+
+export class SuspectLane {
+  private held = 0;
+  private readonly inFlight = new Set<string>();
+
+  constructor(private readonly capacity: number) {
+    if (!Number.isInteger(capacity) || capacity < 1) {
+      throw new Error(`SuspectLane capacity must be a positive integer, got ${capacity}`);
+    }
+  }
+
+  /**
+   * Try to admit a suspect job. On `admitted: true` the caller MUST call
+   * `release(key)` exactly once (in a `finally`) when the run completes.
+   */
+  tryAdmit(key: string): AdmissionResult {
+    if (this.inFlight.has(key)) return { admitted: false, reason: "in_flight" };
+    if (this.held >= this.capacity) return { admitted: false, reason: "lane_full" };
+    this.held++;
+    this.inFlight.add(key);
+    return { admitted: true };
+  }
+
+  /** Release a previously-admitted key. Idempotent for an unknown key. */
+  release(key: string): void {
+    if (this.inFlight.delete(key)) this.held--;
+  }
+
+  /** Number of suspect jobs currently holding a slot (for metrics/tests). */
+  get active(): number {
+    return this.held;
+  }
+}