deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -0,0 +1,1416 @@
+/**
+ * Pooled connection layer for the Postgres runtime scheduler control plane.
+ *
+ * The scheduler's hot path is short, idempotent transactions (submit, claim,
+ * complete, receipts). Three properties matter more than anything else here:
+ *
+ *   1. No connection churn. Every operation runs on a process-singleton
+ *      `pg.Pool` aimed at the *pooled* (PgBouncer transaction-mode) endpoint.
+ *      Opening a fresh `pg.Client` per operation — what the prototype did —
+ *      costs a TLS handshake plus auth round trips per call and collapses
+ *      under burst.
+ *   2. Bounded everything. Client caps, acquire timeouts, query timeouts, and
+ *      an explicit saturation error once the acquire queue backs up, so
+ *      callers shed load instead of stacking unbounded waiters.
+ *   3. Loud failure + fast recovery. A circuit breaker opens after repeated
+ *      connection failures so a dead DB fails calls in microseconds instead
+ *      of stacking 5s connect timeouts, and a half-open trial closes it as
+ *      soon as the DB is back.
+ *
+ * LISTEN/NOTIFY does not work through PgBouncer transaction pooling, so the
+ * notify hub derives the direct (non-pooler) host for its single listener
+ * connection. Notifications are strictly a wakeup hint — the tables remain
+ * the source of truth and every waiter still reads the DB.
+ */
+import { Client, Pool } from 'pg';
+import {
+  RuntimeSchedulerPgBouncerCapacityError,
+  RuntimeSchedulerSubmissionAdmissionError,
+  RuntimeSchedulerSaturatedError,
+} from './scheduler-capacity';
+import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+export {
+  isRuntimeSchedulerCapacityError,
+  parseRuntimeSchedulerCapacityError,
+  RuntimeSchedulerCapacityError,
+  RuntimeSchedulerPgBouncerCapacityError,
+  RuntimeSchedulerSubmissionAdmissionError,
+  RuntimeSchedulerSaturatedError,
+  runtimeSchedulerCapacityRetryAfterMs,
+} from './scheduler-capacity';
+
+const PG_SSL_FILE_PARAMS = ['sslcert', 'sslkey', 'sslrootcert'] as const;
+
+/**
+ * PlanetScale and Neon Postgres URLs can use `sslrootcert=system` to mean "use
+ * the host system CA bundle". `pg-connection-string` treats sslcert/sslkey/
+ * sslrootcert as file paths and tries to open a file literally named `system`,
+ * so normalize that sentinel before handing the URL to node-postgres. Node's
+ * TLS stack uses system roots by default when no custom CA is supplied.
+ */
+export function normalizePostgresConnectionStringForPg(url: string): string {
+  try {
+    const parsed = new URL(url);
+    let changed = false;
+    for (const param of PG_SSL_FILE_PARAMS) {
+      if (parsed.searchParams.get(param)?.trim().toLowerCase() === 'system') {
+        parsed.searchParams.delete(param);
+        changed = true;
+      }
+    }
+    return changed ? parsed.toString() : url;
+  } catch {
+    return url;
+  }
+}
+
+/**
+ * Canonical env var for the workflow runtime control-plane Postgres
+ * (scheduler queue, leases, receipts, outbox). Hard-cutover runtime wiring
+ * must use this name in local dev, CI, Fly, and Vercel.
+ */
+export function resolveWorkflowRuntimePostgresUrl(): string | null {
+  const url = process.env.WORKFLOW_RUNTIME_POSTGRES?.trim() || null;
+  return url ? normalizePostgresConnectionStringForPg(url) : null;
+}
+
+function parsePostgresUrl(url: string): URL | null {
+  try {
+    const parsed = new URL(url);
+    const protocol = parsed.protocol.toLowerCase();
+    if (protocol !== 'postgres:' && protocol !== 'postgresql:') return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function isPlanetScalePostgresUrl(parsed: URL): boolean {
+  const host = parsed.hostname.toLowerCase();
+  return host.includes('psdb.cloud') || host.includes('planetscale');
+}
+
+export function isKnownPooledRuntimePostgresUrl(url: string): boolean {
+  const parsed = parsePostgresUrl(url);
+  if (!parsed) return false;
+  if (isPlanetScalePostgresUrl(parsed)) {
+    return parsed.port === '6432';
+  }
+  const host = parsed.hostname.toLowerCase();
+  return host.includes('-pooler') || parsed.searchParams.has('pool_timeout');
+}
+
+export class RuntimeSchedulerDirectDatabaseUrlError extends Error {
+  constructor() {
+    super(
+      'WORKFLOW_RUNTIME_POSTGRES points at PlanetScale direct Postgres (port 5432). ' +
+        'Runtime scheduler hot paths must use the PgBouncer endpoint (port 6432); ' +
+        'reserve direct connections for explicit session-only operations.',
+    );
+    this.name = 'RuntimeSchedulerDirectDatabaseUrlError';
+  }
+}
+
+export function assertRuntimeSchedulerHotPathUrl(url: string): void {
+  const parsed = parsePostgresUrl(url);
+  if (!parsed) return;
+  if (isPlanetScalePostgresUrl(parsed) && (parsed.port || '5432') === '5432') {
+    throw new RuntimeSchedulerDirectDatabaseUrlError();
+  }
+}
+
+export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
+  /** Per-process client cap. Workers override via options; keep small on serverless. */
+  maxClients: 8,
+  /** Refuse to construct a pool above this — protects the shared PgBouncer/Postgres. */
+  maxClientsHardCap: 32,
+  /** Max time a caller waits for a pooled connection before erroring. */
+  acquireTimeoutMs:
+    RUNTIME_RELIABILITY_POLICY.schedulerDatabase.acquireTimeoutMs,
+  /** Idle pooled connections are recycled after this. */
+  idleTimeoutMs: 30_000,
+  connectTimeoutMs: 5_000,
+  /** Client-side query timeout. Server-side SETs are unsafe through transaction pooling. */
+  queryTimeoutMs: RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs,
+  /** Pending operations beyond one dense-100 burst fail fast instead of growing without bound. */
+  maxAcquireQueueDepth: 128,
+  retry: {
+    attempts: RUNTIME_RELIABILITY_POLICY.schedulerDatabase.retryAttempts,
+    baseDelayMs: RUNTIME_RELIABILITY_POLICY.schedulerDatabase.retryBaseDelayMs,
+    maxDelayMs: RUNTIME_RELIABILITY_POLICY.schedulerDatabase.retryMaxDelayMs,
+  },
+  breaker: {
+    /** Consecutive connection-category failures before the circuit opens. */
+    threshold: 5,
+    /** How long the circuit stays open before a half-open trial. */
+    openMs: RUNTIME_RELIABILITY_POLICY.schedulerDatabase.circuitOpenMs,
+  },
+  /**
+   * Keepalive ping cadence for long-lived worker processes and PgBouncer-style
+   * pooled endpoints.
+   */
+  keepAliveIntervalMs: 60_000,
+} as const;
+
+/**
+ * A control-plane process only needs a few short scheduler transactions at
+ * once. This shared cap keeps horizontally scaled workers and the receipt
+ * gateway from consuming PgBouncer's global client budget independently.
+ */
+export const RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS = 2;
+
+export type RuntimeSchedulerPoolOptions = {
+  url: string;
+  applicationName?: string;
+  maxClients?: number;
+};
+
+export type RuntimeSchedulerQueryClient = {
+  query<Row extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: unknown[],
+  ): Promise<{ rows: Row[] }>;
+};
+
+export class RuntimeSchedulerCircuitOpenError extends Error {
+  constructor(cause: string) {
+    super(
+      `Runtime scheduler DB circuit breaker is open after repeated connection failures: ${cause}`,
+    );
+    this.name = 'RuntimeSchedulerCircuitOpenError';
+  }
+}
+
+type LatencyRing = {
+  samples: number[];
+  next: number;
+};
+
+const LATENCY_RING_SIZE = 512;
+
+function pushSample(ring: LatencyRing, value: number): void {
+  if (ring.samples.length < LATENCY_RING_SIZE) {
+    ring.samples.push(value);
+    return;
+  }
+  ring.samples[ring.next % LATENCY_RING_SIZE] = value;
+  ring.next = (ring.next + 1) % LATENCY_RING_SIZE;
+}
+
+function ringQuantile(ring: LatencyRing, q: number): number | null {
+  if (ring.samples.length === 0) return null;
+  const sorted = [...ring.samples].sort((a, b) => a - b);
+  const idx = Math.min(sorted.length - 1, Math.ceil(q * sorted.length) - 1);
+  return sorted[Math.max(0, idx)] ?? null;
+}
+
+type BreakerState = {
+  consecutiveFailures: number;
+  openedAt: number | null;
+  lastFailureMessage: string;
+  halfOpenInFlight: boolean;
+};
+
+export type RuntimeSchedulerPoolPriority = 'ordinary' | 'critical';
+
+type PoolAdmissionWaiter = {
+  priority: RuntimeSchedulerPoolPriority;
+  resolve: (release: () => void) => void;
+  reject: (error: Error) => void;
+  timeout: ReturnType<typeof setTimeout>;
+  signal?: AbortSignal;
+  onAbort?: () => void;
+};
+
+/**
+ * Critical work may skip one queued launch, but never an unbounded launch
+ * backlog. Alternation keeps already-running Plays moving without turning a
+ * burst of executor-token lookups into starvation for new durable launches.
+ */
+const MAX_CONSECUTIVE_CRITICAL_ADMISSIONS = 1;
+
+/**
+ * Keep priority outside `pg.Pool`: once pg-pool owns a checkout request it is
+ * FIFO and an already-running Play's control-plane request can sit behind an
+ * entire launch burst. This gate admits no more work than the existing client
+ * cap and only changes which bounded waiter receives the next free client.
+ */
+export class RuntimeSchedulerPoolAdmission {
+  readonly #waiting: Record<
+    RuntimeSchedulerPoolPriority,
+    PoolAdmissionWaiter[]
+  > = {
+    ordinary: [],
+    critical: [],
+  };
+  #active = 0;
+  #consecutiveCritical = 0;
+
+  constructor(
+    readonly maxActive: number,
+    readonly maxWaiting: number,
+    readonly maxWaitMs: number = RUNTIME_SCHEDULER_POOL_DEFAULTS.acquireTimeoutMs,
+  ) {}
+
+  acquire(
+    priority: RuntimeSchedulerPoolPriority,
+    signal?: AbortSignal,
+  ): Promise<() => void> {
+    throwIfSchedulerAborted(signal);
+    if (this.#active < this.maxActive && this.waitingCount === 0) {
+      return Promise.resolve(this.#grant());
+    }
+    if (this.waitingCount >= this.maxWaiting) {
+      throw new RuntimeSchedulerSaturatedError(this.waitingCount);
+    }
+    return new Promise((resolve, reject) => {
+      const waiter: PoolAdmissionWaiter = {
+        priority,
+        resolve,
+        reject,
+        signal,
+        timeout: setTimeout(() => {
+          const index = this.#waiting[priority].indexOf(waiter);
+          if (index < 0) return;
+          this.#waiting[priority].splice(index, 1);
+          signal?.removeEventListener('abort', waiter.onAbort!);
+          reject(new RuntimeSchedulerSaturatedError(this.waitingCount));
+        }, this.maxWaitMs),
+      };
+      waiter.onAbort = () => {
+        const index = this.#waiting[priority].indexOf(waiter);
+        if (index < 0) return;
+        this.#waiting[priority].splice(index, 1);
+        clearTimeout(waiter.timeout);
+        reject(schedulerAbortReason(signal!));
+      };
+      signal?.addEventListener('abort', waiter.onAbort, { once: true });
+      this.#waiting[priority].push(waiter);
+      this.#drain();
+    });
+  }
+
+  get waitingCount(): number {
+    return this.#waiting.ordinary.length + this.#waiting.critical.length;
+  }
+
+  #grant(): () => void {
+    this.#active += 1;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      this.#active -= 1;
+      this.#drain();
+    };
+  }
+
+  #drain(): void {
+    while (this.#active < this.maxActive) {
+      const ordinary =
+        this.#consecutiveCritical >= MAX_CONSECUTIVE_CRITICAL_ADMISSIONS
+          ? this.#waiting.ordinary.shift()
+          : undefined;
+      const waiter =
+        ordinary ??
+        this.#waiting.critical.shift() ??
+        this.#waiting.ordinary.shift();
+      if (!waiter) return;
+      clearTimeout(waiter.timeout);
+      waiter.signal?.removeEventListener('abort', waiter.onAbort!);
+      this.#consecutiveCritical =
+        waiter.priority === 'critical' ? this.#consecutiveCritical + 1 : 0;
+      waiter.resolve(this.#grant());
+    }
+  }
+}
+
+type PoolEntry = {
+  pool: Pool;
+  maxClients: number;
+  admission: RuntimeSchedulerPoolAdmission;
+  breaker: BreakerState;
+  stats: {
+    acquires: number;
+    acquireWait: LatencyRing;
+    saturationRejections: number;
+    capacityRejections: number;
+    breakerRejections: number;
+    connectionErrors: number;
+    queryErrors: number;
+    retries: number;
+    peakTotalClients: number;
+    peakWaitingClients: number;
+  };
+};
+
+const poolRegistryKey = Symbol.for(
+  'deepline.runtime-scheduler.postgres-pool.registry',
+);
+
+function getPoolRegistry(): Map<string, PoolEntry> {
+  const globalWithRegistry = globalThis as typeof globalThis & {
+    [poolRegistryKey]?: Map<string, PoolEntry>;
+  };
+  globalWithRegistry[poolRegistryKey] ??= new Map();
+  return globalWithRegistry[poolRegistryKey];
+}
+
+/**
+ * Connection-category errors are safe to retry at the operation level because
+ * every scheduler operation is idempotent (ON CONFLICT guards, lease tokens).
+ */
+export function isConnectionCategoryError(error: unknown): boolean {
+  const seen = new Set<object>();
+  const inspect = (candidate: unknown): boolean => {
+    if (!candidate || typeof candidate !== 'object' || seen.has(candidate)) {
+      return false;
+    }
+    seen.add(candidate);
+
+    if (
+      candidate instanceof AggregateError &&
+      [...candidate.errors].some(inspect)
+    ) {
+      return true;
+    }
+    const cause = (candidate as { cause?: unknown }).cause;
+    if (inspect(cause)) return true;
+
+    const code = (candidate as { code?: unknown }).code;
+    if (typeof code === 'string') {
+      // 08* = connection exceptions; 57P01-57P03 = server shutdown/crash.
+      // 53300 is deliberately NOT here: PgBouncer admission pressure is a
+      // capacity signal, so durable work must defer instead of opening the
+      // connection-failure circuit and burning execution attempts.
+      if (code.startsWith('08')) return true;
+      if (code === '57P01' || code === '57P02' || code === '57P03') {
+        return true;
+      }
+      if (
+        code === 'ECONNRESET' ||
+        code === 'ECONNREFUSED' ||
+        code === 'ETIMEDOUT' ||
+        code === 'EPIPE' ||
+        code === 'ENOTFOUND'
+      ) {
+        return true;
+      }
+    }
+    const message = (candidate as { message?: unknown }).message;
+    if (typeof message === 'string') {
+      return (
+        message.includes('Connection terminated') ||
+        /\bterminating connection\b/i.test(message) ||
+        message.includes('Client has encountered a connection error') ||
+        message.includes('connection timeout') ||
+        message.includes('Query read timeout') ||
+        // This is the scheduler pooler's own upstream-connect failure. It is
+        // observed before a PostgreSQL session exists, so retrying this
+        // idempotent scheduler operation cannot replay customer code. Keep the
+        // normalization at the DB adapter boundary rather than teaching launch
+        // callers to parse an infrastructure string.
+        /\bfailed to connect to upstream\b/i.test(message)
+      );
+    }
+    return false;
+  };
+  return inspect(error);
+}
+
+/**
+ * These PostgreSQL errors abort only the current transaction. Scheduler
+ * operations already carry stable operation identities, so re-running the
+ * whole operation is safe and is materially better than turning a one-off
+ * lock conflict into a failed Play or an unhealthy Machine.
+ */
+export function isRuntimeSchedulerRetryableTransactionError(
+  error: unknown,
+): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const code = (error as { code?: unknown }).code;
+  return code === '40001' || code === '40P01' || code === '55P03';
+}
+
+/**
+ * node-postgres' pool emits this exact lowercase message only when a caller
+ * times out waiting in the local checkout queue. A new physical connection
+ * timing out is rewritten to `Connection terminated due to connection
+ * timeout`, so it must remain a breaker-eligible connection failure.
+ */
+export function isRuntimeSchedulerLocalAcquireTimeout(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    error.message === 'timeout exceeded when trying to connect'
+  );
+}
+
+/**
+ * Connection-capacity exhaustion is a *retryable saturation* signal, not a
+ * broken connection: callers that own durable work must defer instead of
+ * burning an execution attempt or opening the connection-failure circuit.
+ *
+ * Two distinct sources land here:
+ *   - PgBouncer admission pressure (`max_client_conn`) — SQLSTATE 53300, or a
+ *     message mentioning `max_client_conn`.
+ *   - Raw Postgres `max_connections` exhaustion when the runtime DB is reached
+ *     directly (no pooler): also SQLSTATE 53300 (`too_many_connections`), but
+ *     at *connect* time node-postgres does not always preserve `.code`, so the
+ *     Postgres-core message forms are matched too. The canonical form is
+ *     "remaining connection slots are reserved for roles with the SUPERUSER
+ *     attribute"; `too many clients already` is the pgbouncer-side twin.
+ * Kept function name for call-site stability; it now covers both pooler and
+ * direct-Postgres capacity exhaustion.
+ */
+export function isPgBouncerCapacityError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if ((error as { code?: unknown }).code === '53300') return true;
+  const message = (error as { message?: unknown }).message;
+  return (
+    typeof message === 'string' &&
+    /(?:max_client_conn|too many connections|too many clients|connection limit|remaining connection slots|reserved for roles with the superuser|sorry, too many clients)/i.test(
+      message,
+    )
+  );
+}
+
+function recordCapacityRejection(entry: PoolEntry, error: unknown): void {
+  entry.stats.capacityRejections += 1;
+  console.warn('[runtime-scheduler-pool.capacity_rejected]', {
+    error: error instanceof Error ? error.message : String(error),
+    maxClients: entry.maxClients,
+    totalClients: entry.pool.totalCount,
+    idleClients: entry.pool.idleCount,
+    waitingClients: entry.pool.waitingCount,
+  });
+}
+
+function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
+  assertRuntimeSchedulerHotPathUrl(options.url);
+  const maxClients =
+    options.maxClients ?? RUNTIME_SCHEDULER_POOL_DEFAULTS.maxClients;
+  if (maxClients > RUNTIME_SCHEDULER_POOL_DEFAULTS.maxClientsHardCap) {
+    throw new Error(
+      `Runtime scheduler pool maxClients=${maxClients} exceeds the hard cap of ` +
+        `${RUNTIME_SCHEDULER_POOL_DEFAULTS.maxClientsHardCap}. Scale worker replicas ` +
+        'instead of widening a single process pool.',
+    );
+  }
+  const registry = getPoolRegistry();
+  const connectionString = normalizePostgresConnectionStringForPg(options.url);
+  const key = `${connectionString}::${maxClients}`;
+  const existing = registry.get(key);
+  if (existing) {
+    // Next dev can retain the process-global pool across a module hot reload.
+    // Upgrade that live entry in place when this admission layer first lands.
+    existing.admission ??= new RuntimeSchedulerPoolAdmission(
+      maxClients,
+      RUNTIME_SCHEDULER_POOL_DEFAULTS.maxAcquireQueueDepth,
+    );
+    return existing;
+  }
+
+  const pool = new Pool({
+    connectionString,
+    application_name:
+      options.applicationName ?? 'deepline-runtime-scheduler-pool',
+    max: maxClients,
+    idleTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.idleTimeoutMs,
+    connectionTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.acquireTimeoutMs,
+    query_timeout: RUNTIME_SCHEDULER_POOL_DEFAULTS.queryTimeoutMs,
+    keepAlive: true,
+  });
+  const entry: PoolEntry = {
+    pool,
+    maxClients,
+    admission: new RuntimeSchedulerPoolAdmission(
+      maxClients,
+      RUNTIME_SCHEDULER_POOL_DEFAULTS.maxAcquireQueueDepth,
+    ),
+    breaker: {
+      consecutiveFailures: 0,
+      openedAt: null,
+      lastFailureMessage: '',
+      halfOpenInFlight: false,
+    },
+    stats: {
+      acquires: 0,
+      acquireWait: { samples: [], next: 0 },
+      saturationRejections: 0,
+      capacityRejections: 0,
+      breakerRejections: 0,
+      connectionErrors: 0,
+      queryErrors: 0,
+      retries: 0,
+      peakTotalClients: 0,
+      peakWaitingClients: 0,
+    },
+  };
+  pool.on('error', (error) => {
+    // Idle clients dropped by the server (failover, suspend) surface here.
+    // The pool replaces them on next acquire; record and move on.
+    entry.stats.connectionErrors += 1;
+    console.warn('[runtime-scheduler-pool.idle_client_error]', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+  });
+  registry.set(key, entry);
+  return entry;
+}
+
+function breakerGate(entry: PoolEntry): void {
+  const { breaker } = entry;
+  if (breaker.openedAt === null) return;
+  const elapsed = Date.now() - breaker.openedAt;
+  if (elapsed < RUNTIME_SCHEDULER_POOL_DEFAULTS.breaker.openMs) {
+    entry.stats.breakerRejections += 1;
+    throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
+  }
+  // Half-open: let exactly one caller through as the trial.
+  if (breaker.halfOpenInFlight) {
+    entry.stats.breakerRejections += 1;
+    throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
+  }
+  breaker.halfOpenInFlight = true;
+}
+
+function recordBreakerSuccess(entry: PoolEntry): void {
+  entry.breaker.consecutiveFailures = 0;
+  entry.breaker.openedAt = null;
+  entry.breaker.halfOpenInFlight = false;
+}
+
+function recordBreakerFailure(entry: PoolEntry, error: unknown): void {
+  const { breaker } = entry;
+  breaker.halfOpenInFlight = false;
+  breaker.consecutiveFailures += 1;
+  breaker.lastFailureMessage =
+    error instanceof Error ? error.message : String(error);
+  if (
+    breaker.consecutiveFailures >=
+    RUNTIME_SCHEDULER_POOL_DEFAULTS.breaker.threshold
+  ) {
+    breaker.openedAt = Date.now();
+  }
+}
+
+function jitteredBackoffMs(attempt: number): number {
+  const base = Math.min(
+    RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.maxDelayMs,
+    RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.baseDelayMs * 2 ** attempt,
+  );
+  return Math.round(base / 2 + Math.random() * (base / 2));
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function schedulerAbortReason(signal: AbortSignal): Error {
+  return signal.reason instanceof Error
+    ? signal.reason
+    : new Error('Runtime scheduler operation was cancelled.');
+}
+
+function throwIfSchedulerAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted) throw schedulerAbortReason(signal);
+}
+
+/**
+ * Run `fn` with a dedicated pooled connection. The connection is held for the
+ * duration of `fn` so multi-statement transactions (BEGIN..COMMIT) stay on one
+ * session. Connection-category failures destroy the bad connection and retry
+ * the whole operation with jittered backoff — safe because scheduler
+ * operations are idempotent by construction.
+ */
+export async function withRuntimeSchedulerClient<T>(
+  options: RuntimeSchedulerPoolOptions,
+  fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  trace?: {
+    runId: string;
+    phase: string;
+    priority?: RuntimeSchedulerPoolPriority;
+  },
+  operationOptions: { signal?: AbortSignal } = {},
+): Promise<T> {
+  const signal = operationOptions.signal;
+  throwIfSchedulerAborted(signal);
+  const entry = getPoolEntry(options);
+  const operationStartedAt = performance.now();
+  const admissionStartedAt = performance.now();
+  const releaseAdmission = await entry.admission.acquire(
+    trace?.priority ?? 'ordinary',
+    signal,
+  );
+  const admissionWaitMs = performance.now() - admissionStartedAt;
+  let lastError: unknown = null;
+  try {
+    throwIfSchedulerAborted(signal);
+    for (
+      let attempt = 0;
+      attempt < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts;
+      attempt += 1
+    ) {
+      throwIfSchedulerAborted(signal);
+      breakerGate(entry);
+      if (
+        entry.pool.waitingCount >=
+        RUNTIME_SCHEDULER_POOL_DEFAULTS.maxAcquireQueueDepth
+      ) {
+        entry.stats.saturationRejections += 1;
+        throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
+      }
+      const acquireStart = performance.now();
+      entry.stats.peakTotalClients = Math.max(
+        entry.stats.peakTotalClients,
+        entry.pool.totalCount,
+      );
+      entry.stats.peakWaitingClients = Math.max(
+        entry.stats.peakWaitingClients,
+        entry.pool.waitingCount,
+      );
+      let poolClient;
+      try {
+        poolClient = await entry.pool.connect();
+      } catch (error) {
+        if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
+          entry.stats.saturationRejections += 1;
+          console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
+            maxClients: entry.maxClients,
+            totalClients: entry.pool.totalCount,
+            idleClients: entry.pool.idleCount,
+            waitingClients: entry.pool.waitingCount,
+          });
+          throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
+        }
+        if (isPgBouncerCapacityError(error)) {
+          recordCapacityRejection(entry, error);
+          throw new RuntimeSchedulerPgBouncerCapacityError(
+            error instanceof Error ? error.message : String(error),
+          );
+        }
+        recordBreakerFailure(entry, error);
+        entry.stats.connectionErrors += 1;
+        lastError = error;
+        if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+          entry.stats.retries += 1;
+          await sleep(jitteredBackoffMs(attempt));
+          throwIfSchedulerAborted(signal);
+          continue;
+        }
+        throw error;
+      }
+      entry.stats.acquires += 1;
+      const acquireWaitMs = performance.now() - acquireStart;
+      entry.stats.peakTotalClients = Math.max(
+        entry.stats.peakTotalClients,
+        entry.pool.totalCount,
+      );
+      pushSample(entry.stats.acquireWait, acquireWaitMs);
+
+      // A checked-out pg client emits 'error' (not via the pool) when its TCP
+      // connection dies while idle — e.g. a worker holding a client across a
+      // long play execution. With no listener that is an uncaught exception
+      // that kills the process (observed: worker crash mid-150k-row run).
+      // Swallow it here; the next query on the dead client fails normally and
+      // flows through the connection-error retry/poison path below.
+      let checkedOutError: unknown = null;
+      const onCheckedOutError = (error: Error) => {
+        checkedOutError = error;
+        entry.stats.connectionErrors += 1;
+        console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
+          error: error.message,
+        });
+      };
+      poolClient.on('error', onCheckedOutError);
+      let clientReleased = false;
+      const releasePoolClient = (error?: Error) => {
+        if (clientReleased) return;
+        clientReleased = true;
+        poolClient.release(error);
+      };
+      const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
+      signal?.addEventListener('abort', onAbort, { once: true });
+
+      let queryCount = 0;
+      let queryMs = 0;
+      try {
+        throwIfSchedulerAborted(signal);
+        const result = await fn({
+          query: async (sql, params) => {
+            throwIfSchedulerAborted(signal);
+            const queryStartedAt = performance.now();
+            queryCount += 1;
+            try {
+              const queryResult = await poolClient.query(sql, params);
+              throwIfSchedulerAborted(signal);
+              return { rows: queryResult.rows };
+            } finally {
+              queryMs += performance.now() - queryStartedAt;
+            }
+          },
+        });
+        throwIfSchedulerAborted(signal);
+        recordBreakerSuccess(entry);
+        signal?.removeEventListener('abort', onAbort);
+        poolClient.removeListener('error', onCheckedOutError);
+        // The connection died mid-fn but fn happened to finish without another
+        // query — destroy the dead connection rather than pooling it.
+        releasePoolClient(
+          checkedOutError instanceof Error ? checkedOutError : undefined,
+        );
+        if (trace) {
+          console.info(
+            '[runtime-scheduler-pool.operation] ' +
+              JSON.stringify({
+                runId: trace.runId,
+                phase: trace.phase,
+                outcome: 'success',
+                attempt: attempt + 1,
+                priority: trace.priority ?? 'ordinary',
+                admissionWaitMs,
+                acquireWaitMs,
+                queryCount,
+                queryMs,
+                totalMs: performance.now() - operationStartedAt,
+              }),
+          );
+        }
+        return result;
+      } catch (error) {
+        if (trace) {
+          console.info(
+            '[runtime-scheduler-pool.operation] ' +
+              JSON.stringify({
+                runId: trace.runId,
+                phase: trace.phase,
+                outcome: 'failed',
+                attempt: attempt + 1,
+                priority: trace.priority ?? 'ordinary',
+                admissionWaitMs,
+                acquireWaitMs,
+                queryCount,
+                queryMs,
+                totalMs: performance.now() - operationStartedAt,
+                errorName: error instanceof Error ? error.name : 'UnknownError',
+              }),
+          );
+        }
+        signal?.removeEventListener('abort', onAbort);
+        poolClient.removeListener('error', onCheckedOutError);
+        if (signal?.aborted) {
+          const cancellation = schedulerAbortReason(signal);
+          releasePoolClient(cancellation);
+          throw cancellation;
+        }
+        if (isPgBouncerCapacityError(error)) {
+          recordCapacityRejection(entry, error);
+          releasePoolClient();
+          throw new RuntimeSchedulerPgBouncerCapacityError(
+            error instanceof Error ? error.message : String(error),
+          );
+        }
+        if (isConnectionCategoryError(error)) {
+          recordBreakerFailure(entry, error);
+          entry.stats.connectionErrors += 1;
+          lastError = error;
+          // Destroy the broken connection instead of returning it to the pool.
+          releasePoolClient(
+            error instanceof Error ? error : new Error(String(error)),
+          );
+          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+            entry.stats.retries += 1;
+            await sleep(jitteredBackoffMs(attempt));
+            throwIfSchedulerAborted(signal);
+            continue;
+          }
+          throw error;
+        }
+        if (isRuntimeSchedulerRetryableTransactionError(error)) {
+          // PostgreSQL has aborted the transaction, so never return this
+          // client to the pool. Re-run the idempotent scheduler operation on a
+          // new session without opening the connection-failure breaker.
+          entry.stats.queryErrors += 1;
+          recordBreakerSuccess(entry);
+          lastError = error;
+          releasePoolClient(
+            error instanceof Error ? error : new Error(String(error)),
+          );
+          if (attempt + 1 < RUNTIME_SCHEDULER_POOL_DEFAULTS.retry.attempts) {
+            entry.stats.retries += 1;
+            await sleep(jitteredBackoffMs(attempt));
+            throwIfSchedulerAborted(signal);
+            continue;
+          }
+          throw error;
+        }
+        // Query-level failure on a healthy connection: count it, return the
+        // connection to the pool, surface the error without retry.
+        entry.stats.queryErrors += 1;
+        recordBreakerSuccess(entry);
+        releasePoolClient();
+        throw error;
+      }
+    }
+    throw lastError instanceof Error
+      ? lastError
+      : new Error('Runtime scheduler pool operation failed after retries.');
+  } finally {
+    releaseAdmission();
+  }
+}
+
+export type RuntimeSchedulerPoolStats = {
+  url: string;
+  maxClients: number;
+  totalCount: number;
+  idleCount: number;
+  waitingCount: number;
+  acquires: number;
+  acquireWaitP50Ms: number | null;
+  acquireWaitP95Ms: number | null;
+  saturationRejections: number;
+  capacityRejections: number;
+  breakerRejections: number;
+  breakerOpen: boolean;
+  connectionErrors: number;
+  queryErrors: number;
+  retries: number;
+  peakTotalClients: number;
+  peakWaitingClients: number;
+};
+
+export function getRuntimeSchedulerPoolStats(): RuntimeSchedulerPoolStats[] {
+  const registry = getPoolRegistry();
+  return [...registry.entries()].map(([key, entry]) => ({
+    url: key.split('::')[0]!.replace(/:\/\/[^@]*@/, '://***@'),
+    maxClients: entry.maxClients,
+    totalCount: entry.pool.totalCount,
+    idleCount: entry.pool.idleCount,
+    waitingCount: entry.admission.waitingCount,
+    acquires: entry.stats.acquires,
+    acquireWaitP50Ms: ringQuantile(entry.stats.acquireWait, 0.5),
+    acquireWaitP95Ms: ringQuantile(entry.stats.acquireWait, 0.95),
+    saturationRejections: entry.stats.saturationRejections,
+    capacityRejections: entry.stats.capacityRejections,
+    breakerRejections: entry.stats.breakerRejections,
+    breakerOpen: entry.breaker.openedAt !== null,
+    connectionErrors: entry.stats.connectionErrors,
+    queryErrors: entry.stats.queryErrors,
+    retries: entry.stats.retries,
+    peakTotalClients: entry.stats.peakTotalClients,
+    peakWaitingClients: entry.stats.peakWaitingClients,
+  }));
+}
+
+export async function closeRuntimeSchedulerPools(): Promise<void> {
+  const registry = getPoolRegistry();
+  const entries = [...registry.values()];
+  registry.clear();
+  await Promise.all([
+    ...entries.map((entry) => entry.pool.end()),
+    closePgNotifyHubs(),
+  ]);
+}
+
+/**
+ * Warm one pooled connection (TLS + auth + first query) so the first real
+ * operation does not pay the cold-connect cost. Also wakes a suspended Neon
+ * compute before user-visible traffic arrives.
+ */
+export async function warmRuntimeSchedulerPool(
+  options: RuntimeSchedulerPoolOptions,
+): Promise<void> {
+  await withRuntimeSchedulerClient(options, async (client) => {
+    await client.query('SELECT 1');
+  });
+}
+
+/**
+ * Periodic SELECT 1 to keep long-lived worker connections healthy. Returns a
+ * stop function.
+ */
+export function startRuntimeSchedulerKeepalive(
+  options: RuntimeSchedulerPoolOptions,
+): () => void {
+  const interval = setInterval(() => {
+    warmRuntimeSchedulerPool(options).catch((error) => {
+      console.warn('[runtime-scheduler-pool.keepalive_failed]', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    });
+  }, RUNTIME_SCHEDULER_POOL_DEFAULTS.keepAliveIntervalMs);
+  interval.unref?.();
+  return () => clearInterval(interval);
+}
+
+/**
+ * LISTEN/NOTIFY is unavailable through PgBouncer transaction pooling, so the
+ * hub connects to the direct host when one is configured. Neon-style `-pooler`
+ * hosts are normalized for the legacy scheduler path; other managed Postgres
+ * URLs are returned unchanged.
+ */
+export function deriveDirectDatabaseUrl(url: string): string {
+  const explicit =
+    process.env.WORKFLOW_RUNTIME_POSTGRES_DIRECT?.trim() ||
+    process.env.DEEPLINE_RUNTIME_DATABASE_DIRECT_URL?.trim();
+  if (explicit) return normalizePostgresConnectionStringForPg(explicit);
+  try {
+    const parsed = new URL(url);
+    if (isPlanetScalePostgresUrl(parsed) && parsed.port === '6432') {
+      parsed.port = '5432';
+    }
+    parsed.hostname = parsed.hostname.replace('-pooler.', '.');
+    return normalizePostgresConnectionStringForPg(parsed.toString());
+  } catch {
+    return normalizePostgresConnectionStringForPg(url);
+  }
+}
+
+/**
+ * The runtime worker needs one direct Postgres session for its optional
+ * LISTEN wake hint. Preview gates must prove that this separate session budget
+ * is available before submitting a workload: the pooled scheduler probe alone
+ * cannot observe direct-session exhaustion.
+ */
+export class RuntimeSchedulerDirectSessionCapacityError extends Error {
+  constructor(reason: string) {
+    super(
+      'Runtime scheduler direct-session capacity is unavailable. ' +
+        'Reclaim stale Preview worker listener owners or use an isolated direct-session endpoint. ' +
+        `Database error: ${reason}`,
+    );
+    this.name = 'RuntimeSchedulerDirectSessionCapacityError';
+  }
+}
+
+export type RuntimeSchedulerDirectSessionClient = Pick<
+  Client,
+  'connect' | 'query' | 'end'
+>;
+
+export async function probeRuntimeSchedulerDirectSession(input: {
+  url: string;
+  applicationName?: string;
+  createClient?: (
+    config: ConstructorParameters<typeof Client>[0],
+  ) => RuntimeSchedulerDirectSessionClient;
+}): Promise<void> {
+  const client = (input.createClient ?? ((config) => new Client(config)))({
+    connectionString: deriveDirectDatabaseUrl(input.url),
+    application_name:
+      input.applicationName ?? 'deepline-runtime-direct-session-preflight',
+  });
+  try {
+    await client.connect();
+    await client.query('select 1');
+  } catch (error) {
+    if (isPgBouncerCapacityError(error)) {
+      throw new RuntimeSchedulerDirectSessionCapacityError(
+        error instanceof Error ? error.message : String(error),
+      );
+    }
+    throw error;
+  } finally {
+    await client.end().catch(() => undefined);
+  }
+}
+
+/**
+ * `standby` means this process lost an optional session advisory-lock election
+ * and deliberately closed its direct connection. It will retry the election
+ * later; callers continue with their durable polling fallback meanwhile.
+ */
+export type PgNotifyHubState = 'connecting' | 'listening' | 'standby' | 'down';
+
+export type PgNotifyHub = {
+  state: () => PgNotifyHubState;
+  subscribe: (handler: (payload: string) => void) => () => void;
+  subscribeState: (handler: (state: PgNotifyHubState) => void) => () => void;
+  stop: () => Promise<void>;
+};
+
+type PgNotifyHubEntry = {
+  hub: PgNotifyHub;
+  refs: number;
+  idleTimer: ReturnType<typeof setTimeout> | null;
+  lastUsedAt: number;
+  closing: Promise<boolean> | null;
+};
+
+export type PgNotifyHubLease = {
+  hub: PgNotifyHub;
+  release: () => void;
+};
+
+const pgNotifyHubsKey = Symbol.for('deepline.runtime-scheduler.pg-notify-hubs');
+const PG_NOTIFY_HUB_IDLE_MS = 30_000;
+const PG_NOTIFY_HUB_MAX_PER_PROCESS = 4;
+
+function getPgNotifyHubs(): Map<string, PgNotifyHubEntry> {
+  const target = globalThis as typeof globalThis & {
+    [pgNotifyHubsKey]?: Map<string, PgNotifyHubEntry>;
+  };
+  target[pgNotifyHubsKey] ??= new Map();
+  return target[pgNotifyHubsKey];
+}
+
+function beginPgNotifyHubClose(
+  hubs: Map<string, PgNotifyHubEntry>,
+  key: string,
+  entry: PgNotifyHubEntry,
+): void {
+  if (entry.closing) return;
+  if (entry.idleTimer) clearTimeout(entry.idleTimer);
+  entry.idleTimer = null;
+  entry.closing = entry.hub
+    .stop()
+    .then(() => {
+      if (hubs.get(key) === entry) hubs.delete(key);
+      return true;
+    })
+    .catch((error) => {
+      // Keep the failed close occupying its slot. Polling remains available,
+      // and opening a replacement could exceed the physical session budget.
+      console.warn('[runtime-scheduler-notify-hub.close_failed]', {
+        channel: key.split('::').at(-1) ?? key,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return false;
+    });
+}
+
+/**
+ * Lease a bounded direct LISTEN connection. When every slot is actively leased,
+ * additional callers receive null and safely use table polling instead.
+ */
+export function acquirePgNotifyHub(
+  input: {
+    url: string;
+    channel: string;
+    applicationName?: string;
+    leaderLockKey?: string;
+  },
+  lifecycle: {
+    idleMs?: number;
+    create?: (input: {
+      url: string;
+      channel: string;
+      applicationName?: string;
+      leaderLockKey?: string;
+    }) => PgNotifyHub;
+  } = {},
+): PgNotifyHubLease | null {
+  const hubs = getPgNotifyHubs();
+  const key = `${input.url}::${input.channel}::${input.leaderLockKey ?? ''}`;
+  let entry = hubs.get(key);
+  if (entry?.closing) return null;
+  if (!entry && hubs.size >= PG_NOTIFY_HUB_MAX_PER_PROCESS) {
+    const idle = [...hubs.entries()]
+      .filter(([, candidate]) => candidate.refs === 0 && !candidate.closing)
+      .sort((left, right) => left[1].lastUsedAt - right[1].lastUsedAt)[0];
+    if (!idle) return null;
+    const [idleKey, idleEntry] = idle;
+    beginPgNotifyHubClose(hubs, idleKey, idleEntry);
+    // Keep the closing entry in the registry until its physical connection is
+    // gone. This caller polls and may acquire the freed slot on its next pass.
+    return null;
+  }
+  if (!entry) {
+    entry = {
+      hub: (lifecycle.create ?? createPgNotifyHub)(input),
+      refs: 0,
+      idleTimer: null,
+      lastUsedAt: Date.now(),
+      closing: null,
+    };
+    hubs.set(key, entry);
+  }
+  if (entry.idleTimer) {
+    clearTimeout(entry.idleTimer);
+    entry.idleTimer = null;
+  }
+  entry.refs += 1;
+  let released = false;
+  return {
+    hub: entry.hub,
+    release: () => {
+      if (released) return;
+      released = true;
+      entry!.refs = Math.max(0, entry!.refs - 1);
+      entry!.lastUsedAt = Date.now();
+      if (entry!.refs > 0) return;
+      entry!.idleTimer = setTimeout(() => {
+        if (entry!.refs > 0 || hubs.get(key) !== entry) return;
+        beginPgNotifyHubClose(hubs, key, entry!);
+      }, lifecycle.idleMs ?? PG_NOTIFY_HUB_IDLE_MS);
+      entry!.idleTimer.unref?.();
+    },
+  };
+}
+
+export async function closePgNotifyHubs(): Promise<void> {
+  const hubs = getPgNotifyHubs();
+  const entries = [...hubs.entries()];
+  for (const [, entry] of entries) {
+    if (entry.idleTimer) clearTimeout(entry.idleTimer);
+  }
+  for (const [key, entry] of entries) {
+    beginPgNotifyHubClose(hubs, key, entry);
+  }
+  const firstResults = await Promise.all(
+    entries.map(([, entry]) => entry.closing ?? Promise.resolve(false)),
+  );
+  const retryErrors: unknown[] = [];
+  await Promise.all(
+    entries.map(async ([key, entry], index) => {
+      if (firstResults[index]) return;
+      try {
+        await entry.hub.stop();
+        if (hubs.get(key) === entry) hubs.delete(key);
+      } catch (error) {
+        retryErrors.push(error);
+      }
+    }),
+  );
+  if (retryErrors.length > 0) {
+    throw new AggregateError(
+      retryErrors,
+      'Failed to close one or more runtime notification hubs.',
+    );
+  }
+}
+
+const NOTIFY_HUB_RECONNECT_BASE_MS = 250;
+const NOTIFY_HUB_CAPACITY_RECONNECT_BASE_MS = 30_000;
+const NOTIFY_HUB_RECONNECT_MAX_MS = 300_000;
+const NOTIFY_HUB_STANDBY_RETRY_MS = 30_000;
+
+export function runtimeSchedulerNotifyReconnectDelayMs(input: {
+  error: unknown;
+  attempt: number;
+  random?: number;
+}): number {
+  const baseMs = isPgBouncerCapacityError(input.error)
+    ? NOTIFY_HUB_CAPACITY_RECONNECT_BASE_MS
+    : NOTIFY_HUB_RECONNECT_BASE_MS;
+  const ceilingMs = Math.min(
+    NOTIFY_HUB_RECONNECT_MAX_MS,
+    baseMs * 2 ** Math.max(0, input.attempt),
+  );
+  const random = Math.min(1, Math.max(0, input.random ?? Math.random()));
+  return Math.max(1, Math.round(ceilingMs / 2 + random * (ceilingMs / 2)));
+}
+
+/**
+ * One long-lived LISTEN connection per (url, channel) with auto-reconnect.
+ * Strictly a wakeup hint: consumers must still read the tables for truth, so
+ * a down hub degrades to polling latency, never to wrong answers.
+ */
+export function createPgNotifyHub(input: {
+  url: string;
+  channel: string;
+  applicationName?: string;
+  /**
+   * When set, only the session holding this advisory lock LISTENs. Losing
+   * candidates close their direct connection immediately and retry later.
+   * This keeps a horizontally scaled worker lane to one steady direct session.
+   */
+  leaderLockKey?: string;
+  /** Test seam for exercising listener lifecycle failures without a real DB. */
+  createClient?: (config: ConstructorParameters<typeof Client>[0]) => Client;
+}): PgNotifyHub {
+  const directUrl = deriveDirectDatabaseUrl(input.url);
+  const handlers = new Set<(payload: string) => void>();
+  const stateHandlers = new Set<(state: PgNotifyHubState) => void>();
+  let state: PgNotifyHubState = 'connecting';
+  let stopped = false;
+  let client: Client | null = null;
+  let clientClosePromise: Promise<void> | null = null;
+  let connectPromise: Promise<void> | null = null;
+  let reconnectAttempt = 0;
+  let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+  const setState = (next: PgNotifyHubState): void => {
+    state = next;
+    for (const handler of stateHandlers) handler(next);
+  };
+
+  const closeClient = (current: Client): Promise<void> => {
+    if (client !== current) return Promise.resolve();
+    if (clientClosePromise) return clientClosePromise;
+    // Defer end() by one microtask so the shared promise is installed before
+    // node-postgres can emit an end event. The slot remains occupied until the
+    // physical close promise succeeds; a rejection retains the client so a
+    // later stop can retry rather than opening a replacement session.
+    const pending = Promise.resolve()
+      .then(() => current.end())
+      .then(() => {
+        if (client === current) client = null;
+      })
+      .finally(() => {
+        if (clientClosePromise === pending) clientClosePromise = null;
+      });
+    clientClosePromise = pending;
+    return pending;
+  };
+
+  const connect = async (): Promise<void> => {
+    if (stopped) return;
+    setState('connecting');
+    const next = (input.createClient ?? ((config) => new Client(config)))({
+      connectionString: directUrl,
+      application_name: input.applicationName ?? 'deepline-runtime-notify-hub',
+    });
+    try {
+      await next.connect();
+      if (input.leaderLockKey) {
+        const election = await next.query<{ acquired: boolean }>(
+          `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+          [input.leaderLockKey],
+        );
+        if (election.rows[0]?.acquired === false) {
+          // Advisory locks are session-scoped. Do not leave a standby process
+          // holding a scarce direct Postgres connection while the elected
+          // worker owns the listener.
+          client = next;
+          setState('standby');
+          await closeClient(next);
+          scheduleStandbyRetry();
+          return;
+        }
+      }
+      await next.query(
+        `LISTEN ${'"' + input.channel.replace(/"/g, '""') + '"'}`,
+      );
+    } catch (error) {
+      client = next;
+      setState('down');
+      try {
+        await closeClient(next);
+      } catch (closeError) {
+        throw new AggregateError(
+          [error, closeError],
+          'Listener setup failed and its database session could not be closed.',
+        );
+      }
+      scheduleReconnect(error);
+      return;
+    }
+    if (stopped) {
+      client = next;
+      await closeClient(next);
+      return;
+    }
+    client = next;
+    setState('listening');
+    reconnectAttempt = 0;
+    next.on('notification', (message) => {
+      if (typeof message.payload === 'string') {
+        for (const handler of handlers) handler(message.payload);
+      }
+    });
+    next.on('error', (error) => {
+      if (stopped || client !== next) return;
+      setState('down');
+      void closeClient(next).then(
+        () => scheduleReconnect(error),
+        (closeError) => {
+          console.warn('[runtime-scheduler-notify-hub.close_failed]', {
+            channel: input.channel,
+            error:
+              closeError instanceof Error
+                ? closeError.message
+                : String(closeError),
+          });
+        },
+      );
+    });
+    next.on('end', () => {
+      if (stopped || client !== next || clientClosePromise) return;
+      client = null;
+      if (state === 'listening')
+        scheduleReconnect(new Error('listener connection ended'));
+    });
+  };
+
+  const scheduleReconnect = (error: unknown): void => {
+    if (stopped) return;
+    setState('down');
+    if (reconnectTimer) return;
+    // A listener is only a wakeup optimization; durable polling remains the
+    // correctness path. When Postgres is at connection capacity, retrying one
+    // direct LISTEN session per worker every ten seconds creates a thundering
+    // herd precisely when the database needs fewer connection attempts. Use a
+    // slower exponential series for capacity and full jitter for every class.
+    const delay = runtimeSchedulerNotifyReconnectDelayMs({
+      error,
+      attempt: reconnectAttempt,
+    });
+    reconnectAttempt += 1;
+    console.warn('[runtime-scheduler-notify-hub.reconnecting]', {
+      channel: input.channel,
+      delayMs: delay,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    reconnectTimer = setTimeout(() => {
+      reconnectTimer = null;
+      startConnect();
+    }, delay);
+    reconnectTimer.unref?.();
+  };
+
+  const scheduleStandbyRetry = (): void => {
+    if (stopped || reconnectTimer) return;
+    reconnectTimer = setTimeout(() => {
+      reconnectTimer = null;
+      startConnect();
+    }, NOTIFY_HUB_STANDBY_RETRY_MS);
+    reconnectTimer.unref?.();
+  };
+
+  const startConnect = (): void => {
+    if (stopped || connectPromise) return;
+    const pending = connect();
+    connectPromise = pending;
+    void pending.then(
+      () => {
+        if (connectPromise === pending) connectPromise = null;
+      },
+      (error) => {
+        if (connectPromise === pending) connectPromise = null;
+        console.warn('[runtime-scheduler-notify-hub.connect_failed]', {
+          channel: input.channel,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      },
+    );
+  };
+
+  startConnect();
+
+  return {
+    state: () => state,
+    subscribe: (handler) => {
+      handlers.add(handler);
+      return () => handlers.delete(handler);
+    },
+    subscribeState: (handler) => {
+      stateHandlers.add(handler);
+      handler(state);
+      return () => stateHandlers.delete(handler);
+    },
+    stop: async () => {
+      stopped = true;
+      if (reconnectTimer) clearTimeout(reconnectTimer);
+      handlers.clear();
+      setState('down');
+      stateHandlers.clear();
+      const pendingConnect = connectPromise;
+      if (pendingConnect) await pendingConnect;
+      if (client) {
+        const current = client;
+        await closeClient(current);
+      }
+    },
+  };
+}