@checkstack/healthcheck-backend 1.16.0 → 1.18.0

package/src/suspect-lane.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "bun:test";
+import { SuspectLane } from "./suspect-lane";
+
+describe("SuspectLane", () => {
+  test("admits up to capacity, then denies with lane_full", () => {
+    const lane = new SuspectLane(2);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("b")).toEqual({ admitted: true });
+    expect(lane.active).toBe(2);
+    expect(lane.tryAdmit("c")).toEqual({ admitted: false, reason: "lane_full" });
+  });
+
+  test("single-flight: a second admit of the same key is denied with in_flight", () => {
+    const lane = new SuspectLane(4);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    expect(lane.active).toBe(1); // the duplicate did not consume a slot
+  });
+
+  test("in_flight is checked before capacity", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    // lane is full AND k is in-flight; in_flight wins (more specific reason).
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+  });
+
+  test("release frees the slot and clears single-flight", () => {
+    const lane = new SuspectLane(1);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true });
+    expect(lane.tryAdmit("k")).toEqual({ admitted: false, reason: "in_flight" });
+    lane.release("k");
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("k")).toEqual({ admitted: true }); // re-admittable
+  });
+
+  test("release is idempotent for unknown/duplicate keys", () => {
+    const lane = new SuspectLane(2);
+    lane.tryAdmit("a");
+    lane.release("a");
+    lane.release("a"); // no-op
+    lane.release("never-admitted"); // no-op
+    expect(lane.active).toBe(0);
+    expect(lane.tryAdmit("a")).toEqual({ admitted: true });
+  });
+
+  test("rejects a non-positive capacity", () => {
+    expect(() => new SuspectLane(0)).toThrow();
+    expect(() => new SuspectLane(-1)).toThrow();
+  });
+});

package/src/suspect-lane.ts

@@ -0,0 +1,61 @@
+/**
+ * Pod-local bulkhead for slot-hogging ("suspect") health-check jobs.
+ *
+ * A correlated outage can turn hundreds of checks into slot-hoggers at once. The
+ * work-conserving bulkhead caps how many suspect jobs may HOLD a worker slot
+ * concurrently on this pod, so healthy checks keep draining. It is deliberately
+ * NON-BLOCKING: a suspect job that cannot be admitted is skipped for this tick
+ * (freeing its worker slot immediately) rather than queued behind the lane -
+ * queueing would just move the starvation back into the shared worker pool.
+ *
+ * Two admission gates, combined:
+ *   - single-flight: at most ONE in-flight run per `(configId, systemId)` job
+ *     key, so a recurring fire that arrives while the previous slow run is still
+ *     executing is dropped instead of stacking up (no buildup).
+ *   - capacity: at most `capacity` suspect jobs holding a slot at once.
+ *
+ * This is pod-local infrastructure, exactly like the queue's own concurrency
+ * semaphore: total suspect concurrency across the cluster is `capacity x pods`,
+ * scaling the same way queue concurrency already does. The CLASSIFICATION that
+ * decides which jobs are suspect derives from durable run history (globally
+ * consistent); only this admission bookkeeping is per-pod.
+ */
+
+export type AdmissionDenial = "in_flight" | "lane_full";
+
+export type AdmissionResult =
+  | { admitted: true }
+  | { admitted: false; reason: AdmissionDenial };
+
+export class SuspectLane {
+  private held = 0;
+  private readonly inFlight = new Set<string>();
+
+  constructor(private readonly capacity: number) {
+    if (!Number.isInteger(capacity) || capacity < 1) {
+      throw new Error(`SuspectLane capacity must be a positive integer, got ${capacity}`);
+    }
+  }
+
+  /**
+   * Try to admit a suspect job. On `admitted: true` the caller MUST call
+   * `release(key)` exactly once (in a `finally`) when the run completes.
+   */
+  tryAdmit(key: string): AdmissionResult {
+    if (this.inFlight.has(key)) return { admitted: false, reason: "in_flight" };
+    if (this.held >= this.capacity) return { admitted: false, reason: "lane_full" };
+    this.held++;
+    this.inFlight.add(key);
+    return { admitted: true };
+  }
+
+  /** Release a previously-admitted key. Idempotent for an unknown key. */
+  release(key: string): void {
+    if (this.inFlight.delete(key)) this.held--;
+  }
+
+  /** Number of suspect jobs currently holding a slot (for metrics/tests). */
+  get active(): number {
+    return this.held;
+  }
+}

package/src/system-health-override.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, it } from "bun:test";
+import {
+  applySystemHealthOverrides,
+  worstHealthStatus,
+  type SystemHealthOverrideInput,
+} from "./system-health-override";
+import type { SystemHealthStatusResponse } from "@checkstack/healthcheck-common";
+
+const at = new Date("2026-07-05T00:00:00.000Z");
+
+function base(
+  status: SystemHealthStatusResponse["status"],
+): SystemHealthStatusResponse {
+  return { status, evaluatedAt: at, checkStatuses: [] };
+}
+
+function override(
+  status: "degraded" | "unhealthy",
+  sourceId: string,
+): SystemHealthOverrideInput {
+  return {
+    status,
+    source: "incident",
+    reason: `Incident ${sourceId}`,
+    sourceId,
+  };
+}
+
+describe("worstHealthStatus", () => {
+  it("orders unhealthy > degraded > healthy", () => {
+    expect(worstHealthStatus("healthy", "degraded")).toBe("degraded");
+    expect(worstHealthStatus("degraded", "unhealthy")).toBe("unhealthy");
+    expect(worstHealthStatus("unhealthy", "degraded")).toBe("unhealthy");
+    expect(worstHealthStatus("healthy", "healthy")).toBe("healthy");
+  });
+});
+
+describe("applySystemHealthOverrides", () => {
+  it("returns the base unchanged (no override field) when there are no overrides", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [],
+    });
+    expect(result.status).toBe("healthy");
+    expect(result.override).toBeUndefined();
+  });
+
+  it("raises a healthy system to the override status and records the source", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [override("unhealthy", "inc-1")],
+    });
+    expect(result.status).toBe("unhealthy");
+    expect(result.override).toEqual({
+      status: "unhealthy",
+      source: "incident",
+      reason: "Incident inc-1",
+      sourceId: "inc-1",
+    });
+  });
+
+  it("applies an override even when the system has no health checks", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"), // no-checks default is healthy
+      overrides: [override("degraded", "inc-2")],
+    });
+    expect(result.status).toBe("degraded");
+    expect(result.override?.status).toBe("degraded");
+  });
+
+  it("keeps the worse HEALTH CHECK status when a check is worse than the override", () => {
+    // Override says degraded, but a check reports unhealthy -> unhealthy wins.
+    const result = applySystemHealthOverrides({
+      base: base("unhealthy"),
+      overrides: [override("degraded", "inc-3")],
+    });
+    expect(result.status).toBe("unhealthy");
+    // The override is still surfaced (it contributed), at its own status.
+    expect(result.override?.status).toBe("degraded");
+  });
+
+  it("picks the worst override when several incidents override the same system", () => {
+    const result = applySystemHealthOverrides({
+      base: base("healthy"),
+      overrides: [
+        override("degraded", "inc-a"),
+        override("unhealthy", "inc-b"),
+        override("degraded", "inc-c"),
+      ],
+    });
+    expect(result.status).toBe("unhealthy");
+    expect(result.override?.sourceId).toBe("inc-b");
+  });
+});

package/src/system-health-override.ts

@@ -0,0 +1,93 @@
+import type {
+  HealthCheckStatus,
+  SystemHealthStatusResponse,
+  SystemHealthOverride,
+} from "@checkstack/healthcheck-common";
+
+/**
+ * Worst-wins ordering for the derived health vocabulary: a higher rank is a
+ * worse status. Mirrors the inline `unhealthy > degraded > healthy` ordering the
+ * per-check rollup uses in `service.ts`, extracted so the incident-override fold
+ * shares the exact same comparison.
+ */
+const HEALTH_RANK: Record<HealthCheckStatus, number> = {
+  healthy: 0,
+  degraded: 1,
+  unhealthy: 2,
+};
+
+/** Returns whichever status is worse (ties return `a`). */
+export function worstHealthStatus(
+  a: HealthCheckStatus,
+  b: HealthCheckStatus,
+): HealthCheckStatus {
+  return HEALTH_RANK[b] > HEALTH_RANK[a] ? b : a;
+}
+
+/**
+ * A non-health-check contribution to a system's health, as consumed by the
+ * fold. Kept source-agnostic (`source`/`sourceId`) so the health plugin does not
+ * hard-code incident semantics; the incident-backed reader maps its rows into
+ * this shape.
+ */
+export interface SystemHealthOverrideInput {
+  /** The status this contributor forces. Never `healthy` in practice. */
+  status: HealthCheckStatus;
+  /** Contributor kind, e.g. "incident". */
+  source: string;
+  /** Human-readable reason, e.g. the incident title. */
+  reason: string;
+  /** Opaque id of the contributing record, e.g. the incident id. */
+  sourceId?: string;
+}
+
+/**
+ * Reads active health overrides for a set of systems. Implemented in the plugin
+ * wiring over the incident RPC; injected into the health service so the service
+ * stays free of a direct incident dependency and tests can stub it.
+ */
+export interface SystemHealthOverrideReader {
+  getActiveOverrides(
+    systemIds: string[],
+  ): Promise<Record<string, SystemHealthOverrideInput[]>>;
+}
+
+/**
+ * Fold a system's active overrides into its health-check-derived status via
+ * worst-wins. The overall status becomes the worst of the checks-derived status
+ * and every override, so an override that raises a system to `degraded` never
+ * masks a health check reporting `unhealthy` (the worse status always wins), and
+ * an override applies even when the system has no health checks at all.
+ *
+ * The worst contributing override (if any) is surfaced on `override` so a UI can
+ * explain why a system reads worse than its checks alone. When the overrides are
+ * empty the base response is returned unchanged (no `override`).
+ */
+export function applySystemHealthOverrides({
+  base,
+  overrides,
+}: {
+  base: SystemHealthStatusResponse;
+  overrides: SystemHealthOverrideInput[];
+}): SystemHealthStatusResponse {
+  if (overrides.length === 0) return base;
+
+  // The override that reads worst wins the `override` slot; a tie keeps the
+  // first (query order), which for incidents is a stable, arbitrary pick.
+  let worst = overrides[0]!;
+  for (const candidate of overrides) {
+    if (HEALTH_RANK[candidate.status] > HEALTH_RANK[worst.status]) {
+      worst = candidate;
+    }
+  }
+
+  const status = worstHealthStatus(base.status, worst.status);
+  const override: SystemHealthOverride = {
+    status: worst.status,
+    source: worst.source,
+    reason: worst.reason,
+    sourceId: worst.sourceId,
+  };
+
+  return { ...base, status, override };
+}