@checkstack/backend-api 0.18.0 → 0.20.0

package/src/advisory-lock.it.test.ts

@@ -0,0 +1,111 @@
+/**
+ * Integration test (real Postgres) for the advisory-lock service.
+ *
+ * This is part of the surgical integration lane (plan §14.4 #1). It pins the
+ * one behaviour fakes cannot model faithfully: Postgres session-level advisory
+ * locks are tied to the DB *connection* that acquired them, so the holding
+ * client must be the same one that releases — and killing the holding
+ * connection must auto-release the lock.
+ *
+ * Gated behind `CHECKSTACK_IT=1` so the default `bun test` never runs it. The
+ * `integration` CI job sets that flag and provides a real Postgres service
+ * container. Connection comes from `CHECKSTACK_IT_PG_URL` (defaulting to the
+ * `docker-compose-dev.yml` Postgres port).
+ */
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { Pool } from "pg";
+import { createAdvisoryLockService } from "./advisory-lock";
+
+const PG_URL =
+  process.env.CHECKSTACK_IT_PG_URL ??
+  "postgres://postgres:postgres@localhost:5432/postgres";
+
+describe.skipIf(!process.env.CHECKSTACK_IT)(
+  "advisory-lock (real Postgres)",
+  () => {
+    let pool: Pool;
+
+    beforeAll(() => {
+      pool = new Pool({ connectionString: PG_URL });
+      // A pooled client can error asynchronously while idle (e.g. its backend
+      // is terminated by the kill test below). pg emits that on the pool; with
+      // no handler it surfaces as an unhandled "Connection terminated
+      // unexpectedly" error that fails the whole test file. Swallowing idle-
+      // client errors is the documented pg pattern - the tests still assert
+      // behaviour through fresh checkouts.
+      pool.on("error", () => {});
+    });
+
+    afterAll(async () => {
+      await pool.end();
+    });
+
+    it("a second tryAcquire of the same key returns null until release", async () => {
+      const service = createAdvisoryLockService(pool);
+      const key = `it-advisory-lock:${crypto.randomUUID()}`;
+
+      const first = await service.tryAcquire(key);
+      expect(first).not.toBeNull();
+
+      // The lock is held — a concurrent acquire of the SAME key must fail.
+      const second = await service.tryAcquire(key);
+      expect(second).toBeNull();
+
+      // After release, a third acquire succeeds.
+      await first?.release();
+      const third = await service.tryAcquire(key);
+      expect(third).not.toBeNull();
+      await third?.release();
+    });
+
+    it("killing the holding connection auto-releases the lock", async () => {
+      const service = createAdvisoryLockService(pool);
+      const key = `it-advisory-lock:${crypto.randomUUID()}`;
+
+      // Acquire on a dedicated client owned by the handle.
+      const held = await service.tryAcquire(key);
+      expect(held).not.toBeNull();
+
+      // While held, the key is unavailable.
+      const blocked = await service.tryAcquire(key);
+      expect(blocked).toBeNull();
+
+      // Terminate ONLY the backend holding the advisory lock - found via
+      // `pg_locks` - from a fresh connection. Dropping that session makes
+      // Postgres auto-release the lock. We deliberately do NOT kill every other
+      // backend (the old approach): that also terminated the pool's idle
+      // connections, whose async "connection terminated" errors flaked the test
+      // and left the pool unusable. The handle holds exactly one advisory lock,
+      // so this targets precisely the holder.
+      const killer = await pool.connect();
+      try {
+        await killer.query(
+          `SELECT pg_terminate_backend(pid)
+             FROM pg_locks
+            WHERE locktype = 'advisory'
+              AND pid <> pg_backend_pid()`,
+        );
+      } finally {
+        killer.release();
+      }
+
+      // The lock should now be acquirable again. Retry briefly because the
+      // server takes a moment to reap the terminated backend's session locks.
+      let reacquired: Awaited<ReturnType<typeof service.tryAcquire>> = null;
+      for (let attempt = 0; attempt < 20 && reacquired === null; attempt++) {
+        reacquired = await service.tryAcquire(key);
+        if (reacquired === null) {
+          await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+      }
+      expect(reacquired).not.toBeNull();
+      await reacquired?.release();
+
+      // The `held` handle still owns its (now-terminated) client. Release it so
+      // the dead client is returned to the pool - otherwise `pool.end()` in
+      // afterAll blocks waiting for the checked-out client to drain. The unlock
+      // query runs against a dead connection and rejects; that's expected.
+      await held?.release().catch(() => {});
+    });
+  },
+);

package/src/advisory-lock.test.ts

@@ -0,0 +1,273 @@
+import { describe, it, expect } from "bun:test";
+import {
+  createAdvisoryLockService,
+  type AdvisoryLockPool,
+  type AdvisoryLockPoolClient,
+} from "./advisory-lock";
+
+/**
+ * Faithful fake of a `pg.Pool` that models Postgres' per-connection
+ * advisory-lock semantics for BOTH lock flavours:
+ *
+ * SESSION locks (`tryAcquire`):
+ *   - A key can be held by at most one connection at a time.
+ *   - `pg_try_advisory_lock` succeeds only if the key is free; it then
+ *     binds the key to the acquiring connection.
+ *   - `pg_advisory_unlock` only frees the key if THIS connection holds it
+ *     (a no-op otherwise) — exactly the bug we are guarding against: an
+ *     unlock issued on a different connection does nothing.
+ *
+ * TRANSACTION locks (`withXactLock`):
+ *   - `pg_advisory_xact_lock` BLOCKS until the key is free, then binds it to
+ *     the acquiring connection's transaction.
+ *   - `COMMIT` / `ROLLBACK` release every xact lock held by that connection
+ *     and wake the next blocked waiter (FIFO) — modelling auto-release and
+ *     the serialization guarantee.
+ *
+ * This lets the tests prove the service keeps acquire + release on ONE
+ * client and that concurrent `withXactLock` callers serialize.
+ */
+interface FakePool extends AdvisoryLockPool {
+  checkedOut: number;
+  released: number;
+}
+
+function makeFakePool(): FakePool {
+  // key -> owning connection id (or absent if free)
+  const heldBy = new Map<string, number>();
+  // xact key -> owning connection id; waiters queued FIFO per key.
+  const xactHeldBy = new Map<string, number>();
+  const xactWaiters = new Map<string, Array<() => void>>();
+  let nextConnId = 0;
+  const counters = { checkedOut: 0, released: 0 };
+
+  // hashtextextended($1, 0) is opaque here — we just key on the raw string,
+  // which is faithful since the SQL is deterministic per key.
+  function keyOf(values: unknown[] | undefined): string {
+    return String(values?.[0]);
+  }
+
+  return {
+    get checkedOut() {
+      return counters.checkedOut;
+    },
+    get released() {
+      return counters.released;
+    },
+    async connect(): Promise<AdvisoryLockPoolClient> {
+      const connId = nextConnId++;
+      counters.checkedOut++;
+      const releaseXactLocks = () => {
+        for (const [key, owner] of [...xactHeldBy.entries()]) {
+          if (owner !== connId) continue;
+          xactHeldBy.delete(key);
+          const next = xactWaiters.get(key)?.shift();
+          if (next) next();
+        }
+      };
+      return {
+        async query<T>(queryText: string, values?: unknown[]) {
+          if (queryText === "BEGIN") return { rows: [] };
+          if (queryText === "COMMIT" || queryText === "ROLLBACK") {
+            releaseXactLocks();
+            return { rows: [] };
+          }
+          const key = keyOf(values);
+          if (queryText.includes("pg_try_advisory_lock")) {
+            const owner = heldBy.get(key);
+            const ok = owner === undefined;
+            if (ok) heldBy.set(key, connId);
+            return { rows: [{ ok } as unknown as T] };
+          }
+          if (queryText.includes("pg_advisory_xact_lock")) {
+            if (!xactHeldBy.has(key)) {
+              xactHeldBy.set(key, connId);
+              return { rows: [] };
+            }
+            // Blocked: enqueue and wait until a holder commits/rolls back.
+            await new Promise<void>((resolve) => {
+              const q = xactWaiters.get(key) ?? [];
+              q.push(() => {
+                xactHeldBy.set(key, connId);
+                resolve();
+              });
+              xactWaiters.set(key, q);
+            });
+            return { rows: [] };
+          }
+          if (queryText.includes("pg_advisory_unlock")) {
+            // Only the owning connection can release — model the leak bug.
+            if (heldBy.get(key) === connId) heldBy.delete(key);
+            return { rows: [{ ok: true } as unknown as T] };
+          }
+          return { rows: [] };
+        },
+        release() {
+          counters.released++;
+        },
+        on() {
+          // The fake never emits async client errors; the real client's
+          // `on('error')` hardening is exercised by the IT against real
+          // Postgres (killing the holding connection).
+        },
+        off() {
+          // Counterpart to `on`; the service detaches its error listener on
+          // release. No-op here since the fake never attaches one.
+        },
+      };
+    },
+  };
+}
+
+describe("createAdvisoryLockService", () => {
+  it("acquire → second acquire fails while held → release → third acquire succeeds", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+
+    const first = await svc.tryAcquire("k");
+    expect(first).not.toBeNull();
+
+    // Held: a second acquire (different pooled connection) must fail.
+    const second = await svc.tryAcquire("k");
+    expect(second).toBeNull();
+
+    // Release on the SAME client that acquired (the bug is release no-op'ing
+    // because it ran on a different connection).
+    await first!.release();
+
+    const third = await svc.tryAcquire("k");
+    expect(third).not.toBeNull();
+    await third!.release();
+  });
+
+  it("returns the client to the pool on both the failed-acquire and release paths", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+
+    const h = await svc.tryAcquire("k");
+    const blocked = await svc.tryAcquire("k"); // fails → must release client
+    expect(blocked).toBeNull();
+    await h!.release();
+
+    // 2 connects (one held+released, one failed+released) => 2 releases.
+    expect(pool.checkedOut).toBe(2);
+    expect(pool.released).toBe(2);
+  });
+
+  it("release is idempotent", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+    const h = await svc.tryAcquire("k");
+    await h!.release();
+    await h!.release(); // no throw, no double client.release
+    expect(pool.released).toBe(1);
+  });
+
+  it("different keys do not block each other", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+    const a = await svc.tryAcquire("a");
+    const b = await svc.tryAcquire("b");
+    expect(a).not.toBeNull();
+    expect(b).not.toBeNull();
+    await a!.release();
+    await b!.release();
+  });
+});
+
+describe("createAdvisoryLockService.withXactLock", () => {
+  const tick = (ms = 5) => new Promise((r) => setTimeout(r, ms));
+
+  it("runs fn, returns its value, and releases the client", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+    const result = await svc.withXactLock({ key: "k", fn: async () => 42 });
+    expect(result).toBe(42);
+    expect(pool.checkedOut).toBe(1);
+    expect(pool.released).toBe(1);
+  });
+
+  it("serializes concurrent calls on the same key (second fn waits for first to commit)", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+    const order: string[] = [];
+
+    let releaseFirst!: () => void;
+    const firstHeld = new Promise<void>((r) => (releaseFirst = r));
+
+    const p1 = svc.withXactLock({
+      key: "k",
+      fn: async () => {
+        order.push("1-start");
+        await firstHeld;
+        order.push("1-end");
+      },
+    });
+
+    // Let p1 acquire the lock before p2 attempts it.
+    await tick();
+    const p2 = svc.withXactLock({
+      key: "k",
+      fn: async () => {
+        order.push("2-start");
+      },
+    });
+
+    // While p1 holds the lock, p2's fn must NOT have started.
+    await tick();
+    expect(order).toEqual(["1-start"]);
+
+    releaseFirst();
+    await Promise.all([p1, p2]);
+    expect(order).toEqual(["1-start", "1-end", "2-start"]);
+    expect(pool.released).toBe(2);
+  });
+
+  it("rolls back and releases the client when fn throws, freeing the lock", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+
+    await expect(
+      svc.withXactLock({
+        key: "k",
+        fn: async () => {
+          throw new Error("boom");
+        },
+      }),
+    ).rejects.toThrow("boom");
+
+    // Lock was released on rollback: a subsequent acquire succeeds promptly.
+    const after = await svc.withXactLock({ key: "k", fn: async () => "ok" });
+    expect(after).toBe("ok");
+    expect(pool.released).toBe(2);
+  });
+
+  it("different keys do not serialize", async () => {
+    const pool = makeFakePool();
+    const svc = createAdvisoryLockService(pool);
+    const started: string[] = [];
+
+    let release!: () => void;
+    const held = new Promise<void>((r) => (release = r));
+
+    const pA = svc.withXactLock({
+      key: "a",
+      fn: async () => {
+        started.push("a");
+        await held;
+      },
+    });
+    await tick();
+    // Key "b" must run even while "a" is still held.
+    await svc.withXactLock({
+      key: "b",
+      fn: async () => {
+        started.push("b");
+      },
+    });
+    expect(started).toContain("b");
+
+    release();
+    await pA;
+  });
+});

package/src/advisory-lock.ts

@@ -0,0 +1,216 @@
+/**
+ * Postgres advisory-lock helpers with correct connection affinity.
+ *
+ * Postgres session-level advisory locks (`pg_try_advisory_lock` /
+ * `pg_advisory_unlock`) are tied to the DB *session* (connection) that
+ * acquired them. The platform runs every plugin query through a
+ * schema-scoped proxy that wraps each statement in its own short
+ * transaction on a connection borrowed from the shared pool and returned
+ * immediately. Acquiring a session lock through that proxy therefore runs
+ * the lock on one pooled connection and the unlock on a *different* one —
+ * so the unlock no-ops and the lock leaks until the original connection is
+ * recycled. This module fixes that two ways:
+ *
+ *   - {@link AdvisoryLockService.tryAcquire} checks out ONE dedicated
+ *     client from the pool, acquires the session lock on it, and returns a
+ *     handle that owns that client. `release()` runs the unlock on the SAME
+ *     client and then returns it to the pool. Use this for long-held locks
+ *     (e.g. an installer election held across a minutes-long `bun install`)
+ *     where a long-open transaction would be unacceptable.
+ *
+ *   - {@link AdvisoryLockService.withXactLock} wraps acquire + work + release
+ *     in a single transaction using `pg_advisory_xact_lock`, which auto-
+ *     releases at COMMIT/ROLLBACK. Use this for SHORT critical sections (e.g. a
+ *     find-then-create dedup) where holding a transaction for the duration is
+ *     fine and the auto-release removes any chance of a leak.
+ *
+ * BOTH run on the service's pool, which MUST be a pool dedicated to advisory
+ * locks (separate from the pool the locked work runs on). A held lock keeps its
+ * connection checked out for the lock's lifetime; if lock and work shared one
+ * pool, concurrency >= pool size would deadlock (every slot a lock-holder
+ * waiting for a work connection). The backend wires this to a dedicated
+ * `lockPool`; that pool also sets `idle_in_transaction_session_timeout` /
+ * `lock_timeout` so a stalled critical section cannot strand a lock forever.
+ *
+ * Keys are arbitrary strings hashed to Postgres' 64-bit lock space via
+ * `hashtextextended(key, 0)`. Callers SHOULD namespace keys (e.g.
+ * `"script-packages.installer"`, `"incident.dedupe:<systemId>"`) since the
+ * advisory-lock space is global to the database server, not schema-scoped.
+ */
+
+/**
+ * Minimal pool surface this module needs. Modelled on `pg.Pool` /
+ * `pg.PoolClient` without importing `pg` directly so the helper stays a
+ * pure type-level contract; the backend wires in the real `adminPool`.
+ */
+export interface AdvisoryLockPoolClient {
+  query<T>(
+    queryText: string,
+    values?: unknown[],
+  ): Promise<{ rows: T[] }>;
+  /** Return the client to the pool. */
+  release(): void;
+  /**
+   * Subscribe to async client errors. A held client (session lock, or an open
+   * xact-lock transaction) is checked out for a while; if its backend dies
+   * (admin termination, failover, network drop) `pg` emits `'error'` on the
+   * client, and an `'error'` with no listener is re-thrown by the EventEmitter
+   * and would crash the pod. We attach a listener so that loss degrades
+   * gracefully instead. Modelled on `pg.Client.on`.
+   */
+  on(event: "error", listener: (err: Error) => void): void;
+  /**
+   * Detach a previously-attached error listener. MUST be called before
+   * returning the client to the pool: pooled clients are reused, so attaching a
+   * fresh listener on every checkout WITHOUT removing it on release leaks one
+   * listener per acquisition on each long-lived physical connection (an
+   * unbounded `MaxListenersExceeded` leak). Modelled on `pg.Client.off`.
+   */
+  off(event: "error", listener: (err: Error) => void): void;
+}
+
+export interface AdvisoryLockPool {
+  connect(): Promise<AdvisoryLockPoolClient>;
+}
+
+/** A held session-level advisory lock that owns its dedicated client. */
+export interface AdvisoryLockHandle {
+  /**
+   * Release the lock (`pg_advisory_unlock` on the SAME client) and return
+   * the client to the pool. Idempotent: a second call is a no-op.
+   */
+  release(): Promise<void>;
+}
+
+export interface AdvisoryLockService {
+  /**
+   * Try to acquire a session-level advisory lock for `key` on a dedicated
+   * pooled client. Returns a handle on success, or `null` if the lock is
+   * already held (by this or another process). The handle owns the client
+   * until `release()` is called, so callers MUST always release in a
+   * `finally`.
+   */
+  tryAcquire(key: string): Promise<AdvisoryLockHandle | null>;
+  /**
+   * Run `fn` while holding a transaction-scoped advisory lock for `key`,
+   * acquired with `pg_advisory_xact_lock` (which BLOCKS until granted) on a
+   * dedicated client from THIS service's pool, and auto-released when that
+   * transaction commits/rolls back after `fn` settles.
+   *
+   * The lock transaction runs on this service's (dedicated lock) pool, while
+   * `fn` does its real work on whatever database it already holds (typically
+   * the shared admin pool). Because the held lock connection and the work
+   * connection come from DIFFERENT pools, the nested acquisition can never
+   * deadlock the work pool. Use this for SHORT critical sections that gate a
+   * read-then-write on another connection.
+   */
+  withXactLock<T>(args: { key: string; fn: () => Promise<T> }): Promise<T>;
+}
+
+/**
+ * Shared no-op `'error'` listener for held clients. A single module-level
+ * reference (rather than a fresh closure per acquisition) is what lets `off`
+ * detach exactly the listener `on` attached, and avoids allocating one per
+ * lock. It captures nothing, so sharing it is safe.
+ */
+const swallowClientError = (): void => {};
+
+/**
+ * Build an {@link AdvisoryLockService} backed by a pool. The backend
+ * provides the real admin pool; tests can provide a faithful fake that
+ * models per-connection session-lock semantics.
+ */
+export function createAdvisoryLockService(
+  pool: AdvisoryLockPool,
+): AdvisoryLockService {
+  return {
+    async tryAcquire(key) {
+      const client = await pool.connect();
+      // A held session lock keeps this client checked out (not idle), so the
+      // pool's own error handler won't cover it. If this backend is terminated
+      // (admin kill / failover) while the lock is held, `pg` emits `'error'`
+      // here; without a listener the process crashes. Swallow it - the session
+      // lock is auto-released server-side when the backend dies, and a stale
+      // `release()` is already a no-op-safe `finally`, so the loss surfaces as
+      // the key simply becoming acquirable again. The listener is removed on
+      // release so it does not accumulate on the reused pooled connection.
+      client.on("error", swallowClientError);
+      const releaseClient = () => {
+        client.off("error", swallowClientError);
+        client.release();
+      };
+      let acquired = false;
+      try {
+        const result = await client.query<{ ok: boolean }>(
+          "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS ok",
+          [key],
+        );
+        acquired = Boolean(result.rows[0]?.ok);
+      } catch (error) {
+        releaseClient();
+        throw error;
+      }
+      if (!acquired) {
+        // Did not get the lock — return the client immediately. (A failed
+        // pg_try_advisory_lock acquires nothing, so there is nothing to
+        // unlock.)
+        releaseClient();
+        return null;
+      }
+
+      let released = false;
+      return {
+        async release() {
+          if (released) return;
+          released = true;
+          try {
+            await client.query(
+              "SELECT pg_advisory_unlock(hashtextextended($1, 0))",
+              [key],
+            );
+          } finally {
+            releaseClient();
+          }
+        },
+      };
+    },
+
+    async withXactLock({ key, fn }) {
+      const client = await pool.connect();
+      // Same rationale as tryAcquire: the lock transaction keeps this client
+      // checked out (idle in transaction) while `fn` runs, so attach an error
+      // listener to survive a backend termination instead of crashing the pod.
+      // Removed in the finally so it does not accumulate on the reused client.
+      client.on("error", swallowClientError);
+      try {
+        await client.query("BEGIN");
+        try {
+          // BLOCKS on this dedicated client until the lock is granted; auto-
+          // released by the COMMIT/ROLLBACK below. `fn`'s own work runs on a
+          // DIFFERENT pool, so no same-pool nested-acquisition deadlock.
+          await client.query(
+            "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",
+            [key],
+          );
+          const result = await fn();
+          await client.query("COMMIT");
+          return result;
+        } catch (error) {
+          // Roll back so the xact lock releases and nothing partial lingers on
+          // this connection before it returns to the pool. Best-effort: if the
+          // backend already died, ROLLBACK throws but release() still frees the
+          // slot and the lock is auto-released server-side.
+          try {
+            await client.query("ROLLBACK");
+          } catch (rollbackError) {
+            void rollbackError;
+          }
+          throw error;
+        }
+      } finally {
+        client.off("error", swallowClientError);
+        client.release();
+      }
+    },
+  };
+}

package/src/collector-strategy.ts

@@ -89,6 +89,15 @@ export interface CollectorStrategy<
     client: TClient;
     pluginId: string;
     runContext?: CollectorRunContext;
+    /**
+     * Resolved secret env for THIS run (the collector's declared
+     * `secretEnv` mapped to values), delivered just-in-time. Injected into
+     * the collector's script execution env and never persisted. Empty /
+     * absent when the collector declares no secrets. The collector is
+     * responsible for masking these values out of its returned output
+     * (source-side defense in depth).
+     */
+    secretEnv?: Record<string, string>;
   }): Promise<CollectorResult<TResult>>;
 
   /**

package/src/core-services.ts

@@ -16,6 +16,7 @@ import type { PluginArtifactStore } from "./plugin-artifact-store";
 import type { EventBus } from "./event-bus-types";
 import type { WebSocketRouteRegistry } from "./ws-registry";
 import type { ReadinessRegistry } from "./readiness-registry";
+import type { AdvisoryLockService } from "./advisory-lock";
 
 export * from "./types";
 
@@ -66,4 +67,10 @@ export const coreServices = {
   readinessRegistry: createServiceRef<ReadinessRegistry>(
     "core.readinessRegistry",
   ),
+  /**
+   * Postgres advisory-lock service backed by a dedicated pooled client, so
+   * session-level locks keep connection affinity across acquire/release.
+   * See {@link AdvisoryLockService}.
+   */
+  advisoryLock: createServiceRef<AdvisoryLockService>("core.advisoryLock"),
 };