@checkstack/incident-backend 1.4.0 → 1.5.0

package/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @checkstack/incident-backend
 
+## 1.5.0
+
+### Minor Changes
+
+- a57f7db: fix(backend): give advisory locks a dedicated connection pool to prevent pool-starvation deadlock
+
+  Both the session-lock service and `withXactLock` HOLD a Postgres connection for
+  the lock's whole lifetime while the gated work runs on a _different_ connection.
+  Both lock and work were drawing from the single shared `adminPool` (which, with
+  no explicit config, defaulted to `max: 10` and `connectionTimeoutMillis: 0` -
+  wait forever). Under concurrency >= pool size, every slot became a lock-holding
+  connection waiting for a work connection that could never free up: a permanent
+  deadlock. It surfaced as all connections stuck `idle in transaction` on
+  `pg_advisory_xact_lock` and every API request hanging into an upstream 502,
+  only after the server had been running long enough to hit that concurrency
+  (e.g. a burst of health-check evaluations or incident dedups).
+
+  Advisory locks now run on a dedicated `lockPool`, separate from `adminPool`, so
+  the acquire graph is acyclic (`lockPool -> adminPool`, never back) and the
+  deadlock class is impossible. `AdvisoryLockService` gains a pooled
+  `withXactLock({ key, fn })` method (lock on the lock pool, work on the admin
+  pool); healthcheck's per-system serializer, incident's dedup-create, and the
+  automation single-mode concurrency lock now use it. The deadlock-prone
+  standalone `withXactLock({ db, ... })` helper is REMOVED.
+
+  Both pools are explicitly configured with `connectionTimeoutMillis` so any
+  future exhaustion fails fast and self-heals instead of hanging, and both get a
+  pool-level `error` handler (an idle pooled client whose backend dies otherwise
+  crashes the pod). The lock pool additionally sets
+  `idle_in_transaction_session_timeout` and `lock_timeout` so a stalled critical
+  section is reaped server-side (auto-releasing the lock) rather than stranding a
+  key forever. The advisory-lock service also now removes its per-client error
+  listener on release (it previously leaked one listener per acquisition on each
+  reused pooled connection - an unbounded `MaxListenersExceeded` leak).
+
+  New env vars (all optional): `DATABASE_POOL_MAX` (default 20),
+  `DATABASE_LOCK_POOL_MAX` (default 10), `DATABASE_POOL_CONNECTION_TIMEOUT_MS`
+  (default 10000), `DATABASE_POOL_IDLE_TIMEOUT_MS` (default 30000),
+  `DATABASE_LOCK_IDLE_TX_TIMEOUT_MS` (default 30000), `DATABASE_LOCK_TIMEOUT_MS`
+  (default 30000). Size pools off
+  `N_pods * (DATABASE_POOL_MAX + DATABASE_LOCK_POOL_MAX) <= max_connections`.
+
+  BREAKING CHANGE: the standalone `withXactLock({ db, key, fn })` export is
+  removed - use `coreServices.advisoryLock.withXactLock({ key, fn })` instead.
+  `IncidentService`'s constructor now requires an `AdvisoryLockService` as its
+  second argument, and the healthcheck `createHealthEntitySerializer` /
+  `executeHealthCheckJob` / `setupHealthCheckWorker` helpers take `advisoryLock`
+  instead of `db` for the serializer.
+
+### Patch Changes
+
+- Updated dependencies [a57f7db]
+  - @checkstack/backend-api@0.20.0
+  - @checkstack/automation-backend@0.4.0
+  - @checkstack/cache-api@0.3.8
+  - @checkstack/catalog-backend@1.3.1
+  - @checkstack/command-backend@0.1.33
+  - @checkstack/integration-backend@0.3.1
+  - @checkstack/cache-utils@0.2.13
+
 ## 1.4.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/incident-backend",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",

package/src/index.ts

@@ -127,6 +127,7 @@ export default createBackendPlugin({
         rpcClient: coreServices.rpcClient,
         signalService: coreServices.signalService,
         cacheManager: coreServices.cacheManager,
+        advisoryLock: coreServices.advisoryLock,
       },
       init: async ({
         logger,
@@ -135,6 +136,7 @@ export default createBackendPlugin({
         rpcClient,
         signalService,
         cacheManager,
+        advisoryLock,
       }) => {
         logger.debug("🔧 Initializing Incident Backend...");
 
@@ -144,6 +146,7 @@ export default createBackendPlugin({
 
         const service = new IncidentService(
           database as SafeDatabase<typeof schema>,
+          advisoryLock,
         );
         // Publish the service for the PLUGIN-BACKED entity `read` accessor
         // (defined in register()). Mutations only run from here onward.
@@ -208,9 +211,14 @@ export default createBackendPlugin({
       // associations) + register subscription specs. Per-system /
       // per-group notification group lifecycle is fully owned by
       // notification-backend now — incident never touches it.
-      afterPluginsReady: async ({ database, logger, rpcClient }) => {
+      afterPluginsReady: async ({
+        database,
+        logger,
+        rpcClient,
+        advisoryLock,
+      }) => {
         const typedDb = database as SafeDatabase<typeof schema>;
-        const service = new IncidentService(typedDb);
+        const service = new IncidentService(typedDb, advisoryLock);
         const notificationClient = rpcClient.forPlugin(NotificationApi);
 
         await Promise.all([

package/src/service.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect, mock, beforeEach } from "bun:test";
+import type { AdvisoryLockService } from "@checkstack/backend-api";
 import { IncidentService } from "./service";
 import {
   incidents,
@@ -7,6 +8,40 @@ import {
   incidentLinks,
 } from "./schema";
 
+/**
+ * In-memory {@link AdvisoryLockService} that faithfully serializes
+ * `withXactLock` calls per key (a racing call on the same key cannot run its
+ * `fn` until the prior call's `fn` settles) — modelling `pg_advisory_xact_lock`
+ * without a real connection. Different keys are independent.
+ */
+function makeFakeAdvisoryLock(): AdvisoryLockService {
+  const tails = new Map<string, Promise<unknown>>();
+  return {
+    tryAcquire: async () => ({ release: async () => {} }),
+    withXactLock<T>({
+      key,
+      fn,
+    }: {
+      key: string;
+      fn: () => Promise<T>;
+    }): Promise<T> {
+      const prior = tails.get(key) ?? Promise.resolve();
+      const result = prior.then(
+        () => fn(),
+        () => fn(),
+      );
+      tails.set(
+        key,
+        result.then(
+          () => undefined,
+          () => undefined,
+        ),
+      );
+      return result;
+    },
+  };
+}
+
 /**
  * Programmable mock DB that records each `select(...).from(...).where(...)`
  * (and optional `.limit(...)`) chain and returns a configurable row array
@@ -48,7 +83,7 @@ describe("IncidentService.hasActiveIncidentWithSuppression", () => {
 
   const setup = (resultsByCall: unknown[][]) => {
     dbHelper = createProgrammableSelectDb(resultsByCall);
-    service = new IncidentService(dbHelper.db as never);
+    service = new IncidentService(dbHelper.db as never, makeFakeAdvisoryLock());
   };
 
   beforeEach(() => {
@@ -134,7 +169,7 @@ describe("IncidentService.hasActiveIncidentWithSuppression", () => {
 describe("IncidentService.getManyEntityStates (plugin-backed entity read)", () => {
   it("returns {} for an empty id set without querying", async () => {
     const dbHelper = createProgrammableSelectDb([]);
-    const service = new IncidentService(dbHelper.db as never);
+    const service = new IncidentService(dbHelper.db as never, makeFakeAdvisoryLock());
     expect(await service.getManyEntityStates([])).toEqual({});
     expect(dbHelper.getCallCount()).toBe(0);
   });
@@ -153,7 +188,7 @@ describe("IncidentService.getManyEntityStates (plugin-backed entity read)", () =
         { incidentId: "inc-2", systemId: "sys-c" },
       ],
     ]);
-    const service = new IncidentService(dbHelper.db as never);
+    const service = new IncidentService(dbHelper.db as never, makeFakeAdvisoryLock());
     const out = await service.getManyEntityStates(["inc-1", "inc-2", "inc-x"]);
     expect(out).toEqual({
       "inc-1": {
@@ -172,7 +207,7 @@ describe("IncidentService.getManyEntityStates (plugin-backed entity read)", () =
       // incidents query returns nothing → no second query.
       [],
     ]);
-    const service = new IncidentService(dbHelper.db as never);
+    const service = new IncidentService(dbHelper.db as never, makeFakeAdvisoryLock());
     expect(await service.getManyEntityStates(["ghost"])).toEqual({});
     expect(dbHelper.getCallCount()).toBe(1);
   });
@@ -182,7 +217,7 @@ describe("IncidentService.getManyEntityStates (plugin-backed entity read)", () =
       [{ id: "inc-1", status: "monitoring", severity: "critical" }],
       [], // no junction rows
     ]);
-    const service = new IncidentService(dbHelper.db as never);
+    const service = new IncidentService(dbHelper.db as never, makeFakeAdvisoryLock());
     const out = await service.getManyEntityStates(["inc-1"]);
     expect(out["inc-1"]).toEqual({
       status: "monitoring",
@@ -300,7 +335,7 @@ function createDedupFakeDb() {
 describe("IncidentService.createIncidentDedupedForSystem (M3)", () => {
   it("two concurrent dedupe creates for one system open exactly ONE incident", async () => {
     const { db, store } = createDedupFakeDb();
-    const service = new IncidentService(db as never);
+    const service = new IncidentService(db as never, makeFakeAdvisoryLock());
 
     const input = {
       title: "Down",

package/src/service.ts

@@ -1,5 +1,5 @@
 import { eq, and, inArray, ne } from "drizzle-orm";
-import { withXactLock, type SafeDatabase } from "@checkstack/backend-api";
+import type { AdvisoryLockService, SafeDatabase } from "@checkstack/backend-api";
 import * as schema from "./schema";
 import {
   incidents,
@@ -27,7 +27,10 @@ function generateId(): string {
 }
 
 export class IncidentService {
-  constructor(private db: Db) {}
+  constructor(
+    private db: Db,
+    private advisoryLock: AdvisoryLockService,
+  ) {}
 
   /**
    * List incidents with optional filters
@@ -528,15 +531,16 @@ export class IncidentService {
       create: () => Promise<IncidentWithSystems>,
     ) => Promise<IncidentWithSystems> = (create) => create(),
   ): Promise<{ incident: IncidentWithSystems; reused: boolean }> {
-    return withXactLock({
-      db: this.db,
+    return this.advisoryLock.withXactLock({
       key: `incident.dedupe-open-for-system:${dedupeSystemId}`,
-      // The find + create run on `this.db` (the pool), NOT on `tx`. That is
-      // safe here because `pg_advisory_xact_lock` BLOCKS every other holder
-      // of this key until this transaction commits: a racing caller waits
-      // at lock-acquire, so its find can't observe "no open incident" until
-      // ours has already committed the insert. The critical section is thus
-      // serialized by the lock window even though it doesn't ride `tx`.
+      // The find + create deliberately run on `this.db` (the admin pool), NOT
+      // on the lock connection. That is safe because `pg_advisory_xact_lock`
+      // BLOCKS every other holder of this key until this lock transaction
+      // commits: a racing caller waits at lock-acquire, so its find can't
+      // observe "no open incident" until ours has already committed the
+      // insert. Crucially, the lock transaction lives on the DEDICATED lock
+      // pool (see `createAdvisoryLockService(lockPool)`), so holding it open
+      // while the work runs on the admin pool cannot starve the admin pool.
       fn: async () => {
         const existing = await this.findActiveIncidentForSystem(dedupeSystemId);
         if (existing) {