@bevel-software/platform-core-backend 0.15.1 → 0.19.0

package/src/modules/database/__tests__/advisory-lease.test.ts

@@ -0,0 +1,152 @@
+import { describe, it, expect } from 'vitest';
+import { AdvisoryLease, AdvisoryLock, type LeaseClient } from '../advisory-lock.js';
+import type { Database } from '../connection.js';
+
+/**
+ * A connection the suite scripts: what the lock query answers, and a way to
+ * make the connection drop out from under the lease. Driven only through the
+ * lease's public surface.
+ */
+function fakeClient(opts: { grants: boolean }) {
+  const listeners = new Map<string, Array<(...args: unknown[]) => void>>();
+  const record = { queries: [] as string[], ended: false };
+  const client: LeaseClient = {
+    async query(text) {
+      record.queries.push(text);
+      if (text.includes('pg_try_advisory_lock')) return { rows: [{ held: opts.grants }] };
+      return { rows: [] };
+    },
+    async end() {
+      record.ended = true;
+    },
+    on(event, listener) {
+      listeners.set(event, [...(listeners.get(event) ?? []), listener]);
+      return client;
+    },
+  };
+  const drop = () => {
+    for (const l of listeners.get('error') ?? []) l(new Error('connection terminated'));
+  };
+  return { client, record, drop };
+}
+
+const db = {} as Database;
+
+describe('AdvisoryLease', () => {
+  it('holds the lease once granted, and keeps the connection open to hold it', async () => {
+    const conn = fakeClient({ grants: true });
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client });
+
+    expect(lease.held).toBe(false);
+    expect(await lease.tryAcquire()).toBe(true);
+    expect(lease.held).toBe(true);
+    // A session lock lives on its connection; ending it would release the lock.
+    expect(conn.record.ended).toBe(false);
+    // Asking again while held costs nothing and opens nothing.
+    expect(await lease.tryAcquire()).toBe(true);
+  });
+
+  it('opens nothing lasting when another process holds it', async () => {
+    const conn = fakeClient({ grants: false });
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client });
+
+    expect(await lease.tryAcquire()).toBe(false);
+    expect(lease.held).toBe(false);
+    // The connection that asked and was refused is not kept around.
+    expect(conn.record.ended).toBe(true);
+  });
+
+  it('releases by unlocking and ending its connection, and is idempotent', async () => {
+    const conn = fakeClient({ grants: true });
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client });
+    await lease.tryAcquire();
+
+    await lease.release();
+    expect(lease.held).toBe(false);
+    expect(conn.record.queries.some((q) => q.includes('pg_advisory_unlock'))).toBe(true);
+    expect(conn.record.ended).toBe(true);
+
+    await expect(lease.release()).resolves.toBeUndefined();
+    await expect(new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client }).release()).resolves.toBeUndefined();
+  });
+
+  it('reports the lease lost the moment its connection drops, before any listener runs', async () => {
+    const conn = fakeClient({ grants: true });
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client });
+    await lease.tryAcquire();
+    let heldWhenTold: boolean | null = null;
+    lease.onLost(() => {
+      heldWhenTold = lease.held;
+    });
+
+    conn.drop();
+
+    // The listener must be able to trust `held` — a holder that keeps working
+    // after this is the double-worker case the lease exists to prevent.
+    expect(heldWhenTold).toBe(false);
+    expect(lease.held).toBe(false);
+    // Losing it is not releasing it: the same lease can be asked for again.
+    const again = fakeClient({ grants: true });
+    const release = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => again.client });
+    expect(await release.tryAcquire()).toBe(true);
+  });
+
+  it('a connection that drops before the grant is not a lease, and not a loss either', async () => {
+    const conn = fakeClient({ grants: true });
+    const failing: LeaseClient = {
+      ...conn.client,
+      async query() {
+        // The drop lands while the lock query is in flight, as a 'error' on
+        // the client and a rejection of the query.
+        conn.drop();
+        throw new Error('connection terminated');
+      },
+    };
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => failing });
+    let told = 0;
+    lease.onLost(() => {
+      told += 1;
+    });
+
+    await expect(lease.tryAcquire()).rejects.toThrow('connection terminated');
+    expect(lease.held).toBe(false);
+    expect(told).toBe(0);
+  });
+
+  it('a stale connection ending later cannot clear a lease held on a new one', async () => {
+    const first = fakeClient({ grants: true });
+    const second = fakeClient({ grants: true });
+    const clients = [first.client, second.client];
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => clients.shift()! });
+    await lease.tryAcquire();
+    first.drop();
+    expect(lease.held).toBe(false);
+    expect(await lease.tryAcquire()).toBe(true);
+    let told = 0;
+    lease.onLost(() => {
+      told += 1;
+    });
+
+    // The first connection's own 'end', delivered late.
+    first.drop();
+
+    expect(lease.held).toBe(true);
+    expect(told).toBe(0);
+  });
+
+  it('does not report a loss for a release it performed itself', async () => {
+    const conn = fakeClient({ grants: true });
+    const lease = new AdvisoryLease(db, AdvisoryLock.CommitWorker, { connect: async () => conn.client });
+    await lease.tryAcquire();
+    let told = 0;
+    lease.onLost(() => {
+      told += 1;
+    });
+
+    await lease.release();
+    // A real client emits 'end' as it closes; that must read as our own doing.
+    conn.drop();
+
+    expect(told).toBe(0);
+  });
+});

package/src/modules/database/__tests__/advisory-lock.test.ts

@@ -0,0 +1,147 @@
+import { describe, it, expect } from 'vitest';
+import { AdvisoryLock, withAdvisoryLock } from '../advisory-lock.js';
+import type { Database } from '../connection.js';
+
+/**
+ * A pool whose connections record what was asked of them. The SQL a lock
+ * issues IS its observable behaviour — there is nothing else to see from
+ * outside — so these suites drive `withAdvisoryLock` with a crafted pool and
+ * read back the statements and the release.
+ */
+interface RecordedClient {
+  queries: string[];
+  released: boolean;
+  releasedWith: Error | undefined;
+}
+
+function fakePool(opts: { failOn?: (sql: string) => Error; hangOn?: (sql: string) => boolean } = {}) {
+  const clients: RecordedClient[] = [];
+  const pool = {
+    async connect() {
+      const record: RecordedClient = { queries: [], released: false, releasedWith: undefined };
+      clients.push(record);
+      return {
+        async query(sql: string) {
+          record.queries.push(sql);
+          // A statement the database never answers: the promise stays pending.
+          if (opts.hangOn?.(sql)) return new Promise<never>(() => undefined);
+          const failure = opts.failOn?.(sql);
+          if (failure) throw failure;
+          return { rows: [] };
+        },
+        release(err?: Error) {
+          record.released = true;
+          record.releasedWith = err;
+        },
+      };
+    },
+  };
+  return { clients, db: { $client: pool } as unknown as Database };
+}
+
+/** The error shape `pg` raises when `lock_timeout` expires. */
+function lockTimeoutError(): Error {
+  return Object.assign(new Error('canceling statement due to lock timeout'), { code: '55P03' });
+}
+
+describe('withAdvisoryLock', () => {
+  it('holds the lock across the callback and releases it by ending the transaction', async () => {
+    const { clients, db } = fakePool();
+    let queriesWhenBodyRan: string[] = [];
+
+    const result = await withAdvisoryLock(db, AdvisoryLock.Migrations, async () => {
+      queriesWhenBodyRan = [...clients[0].queries];
+      return 'migrated';
+    });
+
+    expect(result).toBe('migrated');
+    // The lock is taken before the body starts...
+    expect(queriesWhenBodyRan.some((q) => q.includes('pg_advisory_xact_lock'))).toBe(true);
+    expect(queriesWhenBodyRan).not.toContain('rollback');
+    // ...and the transaction that holds it ends after the body, which is what
+    // releases both the lock and the transaction-local lock_timeout.
+    expect(clients[0].queries.at(-1)).toBe('rollback');
+    expect(clients[0].released).toBe(true);
+    expect(clients[0].releasedWith).toBeUndefined();
+  });
+
+  it('bounds the wait rather than blocking the boot forever', async () => {
+    const { clients, db } = fakePool();
+
+    await withAdvisoryLock(db, AdvisoryLock.Migrations, async () => undefined, { waitMs: 1234 });
+
+    // Transaction-local, so the setting leaves with the transaction and the
+    // connection goes back to the pool as it arrived.
+    const setTimeoutCall = clients[0].queries.find((q) => q.includes('set_config'));
+    expect(setTimeoutCall).toContain('lock_timeout');
+    expect(setTimeoutCall).toContain('true');
+  });
+
+  it('releases the lock when the callback throws, and propagates the failure', async () => {
+    const { clients, db } = fakePool();
+    const boom = new Error('migration 0004 failed');
+
+    await expect(
+      withAdvisoryLock(db, AdvisoryLock.Migrations, async () => {
+        throw boom;
+      }),
+    ).rejects.toBe(boom);
+
+    expect(clients[0].queries.at(-1)).toBe('rollback');
+    expect(clients[0].released).toBe(true);
+  });
+
+  it('never runs the callback when the lock cannot be taken in time', async () => {
+    const { clients, db } = fakePool({
+      failOn: (sql) => (sql.includes('pg_advisory_xact_lock') ? lockTimeoutError() : undefined!),
+    });
+    let ran = false;
+
+    await expect(
+      withAdvisoryLock(db, AdvisoryLock.Migrations, async () => {
+        ran = true;
+      }),
+    ).rejects.toThrow(/Migrations advisory lock/);
+
+    expect(ran).toBe(false);
+    // Still cleaned up: a failed acquire leaves an open transaction behind
+    // otherwise, and the connection would carry it back into the pool.
+    expect(clients[0].queries.at(-1)).toBe('rollback');
+    expect(clients[0].released).toBe(true);
+  });
+
+  it('fails the boot, and destroys the connection without rolling back, when the database stops answering', async () => {
+    // The socket is up, the server is gone: `begin` never returns. Waiting on
+    // a rollback would queue behind it and hang the boot after all.
+    const { clients, db } = fakePool({ hangOn: (sql) => sql === 'begin' });
+    let ran = false;
+    await expect(
+      withAdvisoryLock(
+        db,
+        AdvisoryLock.Migrations,
+        async () => {
+          ran = true;
+        },
+        { waitMs: 20 },
+      ),
+    ).rejects.toThrow(/did not answer/);
+    expect(ran).toBe(false);
+    expect(clients[0].queries).not.toContain('rollback');
+    expect(clients[0].released).toBe(true);
+    expect(clients[0].releasedWith).toBeInstanceOf(Error);
+  });
+
+  it('destroys a connection it could not roll back instead of pooling it', async () => {
+    const { clients, db } = fakePool({
+      failOn: (sql) => (sql === 'rollback' ? new Error('connection terminated') : undefined!),
+    });
+
+    await withAdvisoryLock(db, AdvisoryLock.Migrations, async () => undefined);
+
+    // Releasing with an error is how `pg` is told to discard the connection.
+    // Ending its session is also what actually releases a lock we may still
+    // be holding, so this path is the fallback for the rollback itself failing.
+    expect(clients[0].released).toBe(true);
+    expect(clients[0].releasedWith).toBeInstanceOf(Error);
+  });
+});

package/src/modules/database/advisory-lock.ts

@@ -0,0 +1,346 @@
+import pg from 'pg';
+import type { Database } from './connection.js';
+
+/**
+ * Postgres advisory locks — the one mechanism by which two Hexis processes
+ * agree that only one of them may do a thing at a time.
+ *
+ * WHY THIS EXISTS. Hexis is single-replica by design (see the notes in
+ * `event-bus.ts` and `pending-commits.worker.ts`), but
+ * nothing enforces it, and the deployment topology guarantees an overlap on
+ * every redeploy: the reverse-proxy deployments this project targets start the
+ * replacement container while the outgoing one is still running — which is
+ * exactly why `docker-compose.yml` publishes no fixed host port. For the
+ * length of that window two processes share one database and one set of
+ * volumes.
+ *
+ * Drizzle's Postgres migrator takes no lock of its own. It reads the newest
+ * applied migration, then opens a transaction and applies everything newer, so
+ * two boots read the same watermark and both apply. With an unguarded
+ * migration (0002 onward are plain DDL) the second boot fails on an
+ * already-applied `ALTER TABLE` and the container crash-loops; with a guarded
+ * one it instead writes a second ledger row for the same migration. A lock
+ * around the runner removes both outcomes, which is what every migration tool
+ * that is not drizzle does.
+ *
+ * WHY A TRANSACTION RATHER THAN A SESSION LOCK. `pg_advisory_lock` is
+ * session-scoped: it belongs to the connection that took it and is released by
+ * `pg_advisory_unlock` or by the session ending. Taken through the pool that is
+ * a trap — the connection carrying the lock returns to the pool still holding
+ * it, and the unlock may run on whichever connection the pool hands out next.
+ * `pg_advisory_xact_lock` is held until the transaction ends, so the `rollback`
+ * below releases it whatever happened inside, and a connection that dies
+ * releases it by dying. The same `rollback` reverts the transaction-local
+ * `lock_timeout`, so nothing about this connection is different when it goes
+ * back to the pool.
+ *
+ * The lock is held on its OWN connection while `fn` runs on others. That is
+ * deliberate and sufficient: the lock excludes other PROCESSES, and it does not
+ * need to be the connection doing the work.
+ *
+ * WHY IT DOES NOT WAIT FOREVER. A boot blocked indefinitely behind a wedged
+ * lock holder is worse than a boot that fails: the container's restart policy
+ * is a working retry, and a failed boot is visible. `lock_timeout` bounds the
+ * wait and surfaces a named error.
+ *
+ * The one caveat worth knowing: the lock connection sits idle in a transaction
+ * for as long as `fn` runs. A server configured with
+ * `idle_in_transaction_session_timeout` shorter than a migration run would kill
+ * it and drop the lock. The default is disabled, and migrations here run in
+ * seconds.
+ */
+
+/**
+ * The first argument to every `pg_advisory_*` call below: a namespace shared by
+ * every lock this application takes, so that an advisory lock belonging to
+ * something else on the same database cannot collide with one of ours by
+ * choosing the same number. `HEXI` read as ASCII bytes — stable, inside
+ * `int4`, and recognisable in `pg_locks` when someone is working out who holds
+ * what.
+ */
+const LOCK_NAMESPACE = 0x48455849;
+
+/**
+ * The locks this application takes, one constant per concern. A new entry is a
+ * new claim that two processes must not do something simultaneously, so it
+ * belongs here beside the others rather than as a number at a call site.
+ */
+export const AdvisoryLock = {
+  /**
+   * Serializes every schema migration across processes — core's and a
+   * distribution's alike, under ONE lock: the enterprise migrations reference
+   * core's tables, so two boots overlapping with one lock each could still
+   * run core's DDL while the other ran enterprise's over it.
+   */
+  Migrations: 1,
+  /**
+   * Held by the ONE process allowed to drain the commit queue — see
+   * {@link AdvisoryLease}. Row-level `SKIP LOCKED` already stops two workers
+   * claiming the same row; it does nothing about two workers running git in
+   * the same working tree on a shared volume, which is what this excludes.
+   */
+  CommitWorker: 3,
+} as const;
+
+export type AdvisoryLockId = (typeof AdvisoryLock)[keyof typeof AdvisoryLock];
+
+/** How long to wait for the lock before giving up and failing the boot. */
+const DEFAULT_WAIT_MS = 60_000;
+
+/** Postgres raises this when `lock_timeout` expires while waiting for a lock. */
+const LOCK_NOT_AVAILABLE = '55P03';
+
+function isLockTimeout(err: unknown): boolean {
+  return (
+    typeof err === 'object' &&
+    err !== null &&
+    (err as { code?: unknown }).code === LOCK_NOT_AVAILABLE
+  );
+}
+
+/** A statement the database did not answer in time: the connection is presumed wedged. */
+export class WireTimeoutError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'WireTimeoutError';
+  }
+}
+
+/** `work`, or a {@link WireTimeoutError} naming `what` once `ms` have passed without an answer. */
+async function bounded<T>(work: Promise<T>, ms: number, what: string): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new WireTimeoutError(`Timed out after ${ms}ms ${what} — the database did not answer.`)), ms);
+    timer.unref?.();
+  });
+  try {
+    return await Promise.race([work, deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** The name of a lock, for an error message a human has to act on. */
+function lockName(lock: AdvisoryLockId): string {
+  const entry = Object.entries(AdvisoryLock).find(([, value]) => value === lock);
+  return entry ? entry[0] : String(lock);
+}
+
+/**
+ * Run `fn` while holding the advisory lock `lock`, waiting up to `waitMs` for
+ * it. The lock is released however `fn` ends, and the connection carrying it
+ * goes back to the pool in the state it arrived in.
+ *
+ * Throws without running `fn` when the lock cannot be taken in time — the
+ * caller is expected to let that fail the boot.
+ */
+export async function withAdvisoryLock<T>(
+  db: Database,
+  lock: AdvisoryLockId,
+  fn: () => Promise<T>,
+  opts: { waitMs?: number } = {},
+): Promise<T> {
+  const waitMs = opts.waitMs ?? DEFAULT_WAIT_MS;
+  const client = await db.$client.connect();
+  // A statement that did not answer in time is STILL RUNNING on this
+  // connection — `bounded` gives up waiting, it cannot take the statement
+  // back — and every later statement on it, the rollback included, queues
+  // behind it. So a connection that wedged is remembered here and destroyed
+  // in the `finally` instead of being rolled back: ending its session is
+  // what releases whatever it held, and nothing waits on it again.
+  let wedged: Error | null = null;
+  const onWire = async <R>(work: Promise<R>, what: string): Promise<R> => {
+    try {
+      return await bounded(work, waitMs, what);
+    } catch (err) {
+      if (err instanceof WireTimeoutError) wedged = err;
+      throw err;
+    }
+  };
+  try {
+    // `lock_timeout` bounds the wait for the lock, once it is set. The two
+    // statements before it run on a connection that may be half-dead — the
+    // socket open, the server gone — and would otherwise wait forever, with
+    // the boot behind them. So they carry a deadline of their own, and a
+    // connection that cannot answer them fails the boot as loudly as one that
+    // cannot take the lock.
+    await onWire(client.query('begin'), 'beginning the lock transaction');
+    try {
+      // `set_config` rather than `SET LOCAL`, which cannot take a bind
+      // parameter. `true` makes the setting transaction-local, so the
+      // `rollback` below reverts it.
+      await onWire(client.query("select set_config('lock_timeout', $1, true)", [String(waitMs)]), 'setting lock_timeout');
+      await client.query('select pg_advisory_xact_lock($1, $2)', [LOCK_NAMESPACE, lock]);
+    } catch (err) {
+      if (isLockTimeout(err)) {
+        throw new Error(
+          `Timed out after ${waitMs}ms waiting for the ${lockName(lock)} advisory lock. ` +
+            'Another process is holding it — usually the outgoing container of a redeploy, ' +
+            'in which case starting again succeeds once it exits.',
+          { cause: err },
+        );
+      }
+      throw err;
+    }
+    return await fn();
+  } finally {
+    if (wedged) {
+      // Not rolled back: a rollback would queue behind the statement that
+      // never answered, and the boot would hang on it after all. Destroyed
+      // — `release(err)` ends the session — which is what lets go of
+      // whatever the connection held.
+      client.release(wedged);
+    } else {
+      // Ends the transaction, which releases the lock and reverts
+      // `lock_timeout`. A connection that cannot be rolled back is destroyed
+      // rather than returned to the pool: ending its session is what releases
+      // the lock, and a connection in an unknown transaction state is no use
+      // to anyone else.
+      let rollbackError: Error | undefined;
+      try {
+        await client.query('rollback');
+      } catch (err) {
+        rollbackError = err instanceof Error ? err : new Error(String(err));
+      }
+      client.release(rollbackError);
+    }
+  }
+}
+
+/** The slice of a `pg.Client` the lease uses — so a suite can hand in a double. */
+export interface LeaseClient {
+  query(text: string, values?: unknown[]): Promise<{ rows: Array<Record<string, unknown>> }>;
+  end(): Promise<void>;
+  on(event: 'error' | 'end', listener: (...args: unknown[]) => void): unknown;
+}
+
+export interface AdvisoryLeaseOptions {
+  /** How the lease opens its connection. Default: a `pg.Client` on the pool's connection string. */
+  connect?: () => Promise<LeaseClient>;
+  /** Bound on the connect and on each query, so an unreachable or half-dead database fails rather than waits. Default 10s. */
+  connectTimeoutMs?: number;
+}
+
+/**
+ * A SESSION-scoped advisory lock held for as long as this process wants it:
+ * the singleton lease. `tryAcquire` never waits — it answers whether this
+ * process is the holder — and the lock lives on a connection of its own for
+ * the process's lifetime, released by {@link release} or by that connection
+ * ending, however it ends.
+ *
+ * WHY ITS OWN CONNECTION, AND WHY THAT IS FINE. A session lock belongs to the
+ * connection that took it, so it cannot ride the pool (see the transaction
+ * form above for what goes wrong when it does). This connection is therefore
+ * opened once and kept, which is the one shape of resource the closing-owner
+ * rule exempts when it is said out loud: its lifetime is intentionally the
+ * process's. The owner that closes it is the shutdown sequence, through
+ * `release`, and if the process dies without one the session dies with it
+ * and Postgres releases the lock — which is exactly how the NEXT process gets
+ * to take it.
+ *
+ * LOSS IS A FIRST-CLASS EVENT. A connection can drop (a database restart, a
+ * network blip) while this process believes it holds the lease; from that
+ * moment another process may hold it too. So the lease watches its connection
+ * and tells its listeners the instant the holding session is gone, and
+ * `held` flips false before any of them run — a holder that keeps working
+ * after that call is the double-worker case this exists to prevent.
+ */
+export class AdvisoryLease {
+  private client: LeaseClient | null = null;
+  private heldFlag = false;
+  private readonly lostListeners: Array<() => void> = [];
+
+  constructor(
+    private readonly db: Database,
+    private readonly lock: AdvisoryLockId,
+    private readonly opts: AdvisoryLeaseOptions = {},
+  ) {}
+
+  /** Whether this process holds the lease right now. */
+  get held(): boolean {
+    return this.heldFlag;
+  }
+
+  /** Called when the holding connection ends without {@link release} — the lease is gone. */
+  onLost(listener: () => void): void {
+    this.lostListeners.push(listener);
+  }
+
+  /**
+   * Take the lease if nobody holds it. Returns at once either way: `true` and
+   * this process is the holder from now on; `false` and nothing is held or
+   * kept open.
+   */
+  async tryAcquire(): Promise<boolean> {
+    if (this.heldFlag) return true;
+    const client = await (this.opts.connect ?? this.defaultConnect)();
+    // Watched BEFORE the lock is asked for — a `pg.Client` throws an 'error'
+    // with no listener at the process — and bound to THIS client: a loss is
+    // only a loss while this is the connection the lease is held on. Before
+    // the grant below the client is not that connection, so a drop during
+    // the query surfaces as the query's own rejection and nothing is marked
+    // held; after a release or a later reacquisition it is not that
+    // connection either, so a stale client's delayed 'end' cannot clear a
+    // lease held on a healthy one.
+    const lost = () => {
+      if (this.client === client) this.markLost();
+    };
+    client.on('error', lost);
+    client.on('end', lost);
+    try {
+      const { rows } = await client.query('select pg_try_advisory_lock($1, $2) as held', [
+        LOCK_NAMESPACE,
+        this.lock,
+      ]);
+      if (rows[0]?.held !== true) {
+        await client.end().catch(() => undefined);
+        return false;
+      }
+      this.client = client;
+      this.heldFlag = true;
+      return true;
+    } catch (err) {
+      await client.end().catch(() => undefined);
+      throw err;
+    }
+  }
+
+  /**
+   * Give the lease up and close its connection. Idempotent, and safe to call
+   * on a lease that was never taken or has already been lost.
+   */
+  async release(): Promise<void> {
+    const client = this.client;
+    this.client = null;
+    this.heldFlag = false;
+    if (!client) return;
+    try {
+      await client.query('select pg_advisory_unlock($1, $2)', [LOCK_NAMESPACE, this.lock]);
+    } catch {
+      // Ending the session below releases the lock regardless.
+    } finally {
+      await client.end().catch(() => undefined);
+    }
+  }
+
+  private markLost(): void {
+    if (!this.heldFlag) return;
+    this.heldFlag = false;
+    this.client = null;
+    for (const listener of this.lostListeners) listener();
+  }
+
+  private readonly defaultConnect = async (): Promise<LeaseClient> => {
+    const timeout = this.opts.connectTimeoutMs ?? 10_000;
+    const client = new pg.Client({
+      connectionString: this.db.$client.options.connectionString,
+      connectionTimeoutMillis: timeout,
+      // The same bound on each query: a connection that opened and then went
+      // half-dead would otherwise hold `tryAcquire` open indefinitely, and
+      // with it the loop that should be asking again in five seconds.
+      query_timeout: timeout,
+    });
+    await client.connect();
+    return client;
+  };
+}

package/src/modules/database/connection.ts

@@ -6,7 +6,15 @@ let db: ReturnType<typeof drizzle<typeof schema>> | null = null;
 
 export function getDb(databaseUrl: string) {
   if (!db) {
-    const pool = new pg.Pool({ connectionString: databaseUrl });
+    // A bounded handshake: `pg` waits forever by default for a NEW connection
+    // to come up, so a database that accepts the socket and never finishes
+    // the handshake would hold a boot (the migration lock is the first thing
+    // that asks) with nothing to say and nothing for the restart policy to
+    // see. Half a minute is generous for a handshake and short enough for a
+    // failed boot to be visible. It bounds only that: a checkout queued
+    // behind a saturated pool, or a query on a connection that stopped
+    // answering, is the caller's to bound — the advisory lock does.
+    const pool = new pg.Pool({ connectionString: databaseUrl, connectionTimeoutMillis: 30_000 });
     db = drizzle(pool, { schema });
   }
   return db;