@dbos-inc/dbos-sdk 4.26.8-preview → 4.26.10

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -100,49 +100,94 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
     }
 }
 exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
-async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
-    let client = null;
+/**
+ * Check out a connection carrying our own error handler, so a socket death while we hold it is
+ * logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
+ * so nothing we attach outlives the checkout and the pool itself is never touched.
+ */
+async function borrowClient(pool, onError) {
+    const client = await pool.connect();
+    // Nothing awaits between the checkout and this attach, nor between the removal and release()
+    // below, so there is no turn of the event loop in which we hold the connection unguarded.
+    client.on('error', onError);
+    const release = client.release.bind(client);
+    client.release = (err) => {
+        client.removeListener('error', onError);
+        release(err);
+    };
+    return client;
+}
+/** Connect to the system database without creating it. The caller releases the client. */
+async function connectToSystemDatabase(sysDbUrl, logger, customPool) {
     if (customPool) {
-        // If a custom pool is passed in, assume the database already exists and create
-        // a client to run migrations.
-        client = await customPool.connect();
+        return await borrowClient(customPool, (err) => logger.warn(`Unexpected error in system database client: ${err}`));
     }
-    else {
-        // Otherwise, create the system database if it does not exist.
-        await (0, database_utils_1.ensurePGDatabase)(sysDbUrl, logger);
-        const sysClient = new pg_1.Client((0, utils_2.getClientConfig)(sysDbUrl));
-        // An 'error' event with no listener would take down the process.
-        sysClient.on('error', (err) => logger.warn(`Unexpected error in system database client: ${err}`));
-        try {
-            await sysClient.connect();
+    const sysClient = new pg_1.Client((0, utils_2.getClientConfig)(sysDbUrl));
+    // An 'error' event with no listener would take down the process.
+    sysClient.on('error', (err) => logger.warn(`Unexpected error in system database client: ${err}`));
+    try {
+        await sysClient.connect();
+    }
+    catch (e) {
+        await sysClient.end().catch(() => { });
+        throw new error_1.DBOSInitializationError(`Unable to connect to system database at ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)}: ${e.message}`, e instanceof Error ? e : undefined);
+    }
+    return sysClient;
+}
+async function releaseSystemDatabaseClient(client, customPool) {
+    try {
+        if (customPool) {
+            client.release();
         }
-        catch (e) {
-            await sysClient.end().catch(() => { });
-            throw new error_1.DBOSInitializationError(`Unable to connect to system database at ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)}: ${e.message}`, e instanceof Error ? e : undefined);
+        else {
+            await client.end();
         }
-        client = sysClient;
     }
+    catch (e) { }
+}
+async function isCockroachDB(client) {
+    const versionRes = await client.query('SELECT version() AS version');
+    return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
+}
+async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
+    if (!customPool) {
+        // A custom pool means the database already exists; otherwise, create it if it does not.
+        await (0, database_utils_1.ensurePGDatabase)(sysDbUrl, logger);
+    }
+    const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
     try {
-        const versionRes = await client.query('SELECT version() AS version');
-        const isCockroach = /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
+        const isCockroach = await isCockroachDB(client);
         await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
             onWarn: (e) => logger.info(e),
             isCockroach,
         });
     }
     finally {
-        try {
-            if (customPool) {
-                client.release();
-            }
-            else {
-                await client.end();
-            }
-        }
-        catch (e) { }
+        await releaseSystemDatabaseClient(client, customPool);
     }
 }
 exports.ensureSystemDatabase = ensureSystemDatabase;
+/** Check the system database is migrated to the version this build requires, creating and changing nothing. */
+async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
+    const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
+    try {
+        const isCockroach = await isCockroachDB(client);
+        const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
+        const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
+        // A database ahead of this build belongs to a newer peer, which the migration runner also tolerates.
+        if (currentVersion < requiredVersion) {
+            throw new error_1.DBOSInitializationError(`System database ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)} is at schema version ${currentVersion}, but this version ` +
+                `of DBOS requires ${requiredVersion}. This process is configured with runMigrations disabled, so it ` +
+                `will not migrate it: either migrate the system database out of band (\`npx dbos schema\`) or launch ` +
+                `with runMigrations enabled.`);
+        }
+        logger.debug(`System database schema version ${currentVersion} satisfies the required version ${requiredVersion}`);
+    }
+    finally {
+        await releaseSystemDatabaseClient(client, customPool);
+    }
+}
+exports.verifySystemDatabase = verifySystemDatabase;
 class NotificationMap {
     map = new Map();
     curCK = 0;
@@ -441,6 +486,8 @@ class SystemDatabase {
     runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
     // Per-partition-key created_at cursors: keep per-key queue order monotonic across batches
     #batchCreatedAtCursors = new Map();
+    // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
+    #destroyed = false;
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
     appName) {
@@ -469,14 +516,20 @@ class SystemDatabase {
         const effectivePoolSize = this.pool.options.max ?? sysDbPoolSize;
         const pollingLimit = pollingConcurrency ?? Math.max(1, Math.floor(effectivePoolSize / 2));
         this.pollLimiter = new utils_1.Semaphore(pollingLimit);
-        this.pool.on('error', (err) => {
-            this.logger.warn(`Unexpected error in pool: ${err}`);
-        });
-        this.pool.on('connect', (client) => {
-            client.on('error', (err) => {
-                this.logger.warn(`Unexpected error in idle client: ${err}`);
+        // Only ever attach listeners to a pool we own; a caller's pool is theirs to instrument. Idle
+        // connections are all this covers, since #connect guards the ones we are holding.
+        if (!this.customPool) {
+            this.pool.on('error', (err) => {
+                this.logger.warn(`Unexpected error in pool: ${err}`);
             });
-        });
+        }
+    }
+    #onClientError = (err) => {
+        this.logger.warn(`Unexpected error on a system database connection: ${err}`);
+    };
+    /** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
+    #connect() {
+        return borrowClient(this.pool, this.#onClientError);
     }
     getSerializer() {
         return this.serializer;
@@ -535,8 +588,10 @@ class SystemDatabase {
             `Either ${takeANewName}, or, if '${current}' was renamed to '${owner}', ` +
             `re-own its rows first with dbos rename-application`);
     }
-    async init() {
-        await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
+    /** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
+    async init(runMigrations = true) {
+        const migrateOrVerify = runMigrations ? ensureSystemDatabase : verifySystemDatabase;
+        await migrateOrVerify(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
         if (this.shouldUseDBNotifications) {
             await this.#listenForNotifications();
             // Push coalesced stream and event notifications off the write path.
@@ -547,6 +602,7 @@ class SystemDatabase {
     async destroy() {
         // Set synchronously, before any await, so no reconnect is scheduled or published after this point.
         this.#notificationsStopped = true;
+        this.#destroyed = true;
         if (this.reconnectTimeout) {
             clearTimeout(this.reconnectTimeout);
             this.reconnectTimeout = null;
@@ -561,45 +617,28 @@ class SystemDatabase {
         if (this.notificationsClient) {
             this.#retireNotificationsClient(this.notificationsClient);
         }
-        await this.pool.end();
+        // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
+        if (!this.customPool) {
+            await this.pool.end();
+        }
     }
     // ==================== Workflow Status ====================
-    async initWorkflowStatus(initStatus, ownerXid) {
-        const client = await this.pool.connect();
+    /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
+    async initWorkflowStatus(initStatus, ownerXid, client) {
+        if (client !== undefined) {
+            return await this.#initWorkflowStatusInternal(client, initStatus, ownerXid);
+        }
+        return await this.initWorkflowStatusStandalone(initStatus, ownerXid);
+    }
+    async initWorkflowStatusStandalone(initStatus, ownerXid) {
+        const client = await this.#connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid);
-            if (resRow.name !== initStatus.workflowName) {
-                const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if (resRow.class_name !== initStatus.workflowClassName) {
-                const msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if ((resRow.config_name || '') !== (initStatus.workflowConfigName || '')) {
-                const msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if ((resRow.queue_name ?? undefined) !== (initStatus.queueName ?? undefined)) {
-                // This is a warning because a different queue name is not necessarily an error.
-                this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
-            }
-            const status = resRow.status;
-            const deadlineEpochMS = resRow.workflow_deadline_epoch_ms ?? undefined;
+            const result = await this.#initWorkflowStatusInternal(client, initStatus, ownerXid);
             // If there is an existing DB record and we aren't here to recover it, leave it be.
-            if (ownerXid !== resRow.owner_xid) {
-                return { status, deadlineEpochMS, shouldExecuteOnThisExecutor: false, serialization: resRow.serialization };
-            }
-            // Upsert above already set executor assignment
-            shouldCommit = true;
-            return {
-                status,
-                deadlineEpochMS,
-                shouldExecuteOnThisExecutor: true,
-                serialization: resRow.serialization,
-            };
+            shouldCommit = result.shouldExecuteOnThisExecutor;
+            return result;
         }
         finally {
             try {
@@ -616,6 +655,34 @@ class SystemDatabase {
             }
         }
     }
+    async #initWorkflowStatusInternal(client, initStatus, ownerXid) {
+        const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid);
+        if (resRow.name !== initStatus.workflowName) {
+            const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if (resRow.class_name !== initStatus.workflowClassName) {
+            const msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if ((resRow.config_name || '') !== (initStatus.workflowConfigName || '')) {
+            const msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if ((resRow.queue_name ?? undefined) !== (initStatus.queueName ?? undefined)) {
+            // This is a warning because a different queue name is not necessarily an error.
+            this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
+        }
+        const status = resRow.status;
+        const deadlineEpochMS = resRow.workflow_deadline_epoch_ms ?? undefined;
+        // The upsert above already set executor assignment for a row we own.
+        return {
+            status,
+            deadlineEpochMS,
+            shouldExecuteOnThisExecutor: ownerXid === resRow.owner_xid,
+            serialization: resRow.serialization,
+        };
+    }
     /** Move claimed workflows that exhausted their attempts off the queue, leaving rows others have moved on alone. */
     async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
         if (workflowIDs.length === 0)
@@ -737,7 +804,7 @@ class SystemDatabase {
             'schedule_name',
             'application_name',
         ];
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Chunk to stay well under the bind-parameter limit.
@@ -775,7 +842,7 @@ class SystemDatabase {
         return inserted;
     }
     async recordWorkflowOutput(workflowID, status) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
         }
@@ -784,7 +851,7 @@ class SystemDatabase {
         }
     }
     async recordWorkflowError(workflowID, status) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error });
         }
@@ -846,7 +913,7 @@ class SystemDatabase {
             return status ? JSON.stringify(status) : null;
         };
         if (callerID && callerFN) {
-            const client = await this.pool.connect();
+            const client = await this.#connect();
             try {
                 // Check if the operation has been done before for OAOO (only do this inside a workflow).
                 const json = await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus);
@@ -882,7 +949,7 @@ class SystemDatabase {
     }
     // Only used in tests
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await this.updateWorkflowStatus(client, workflowID, status, {
                 update: {
@@ -899,7 +966,7 @@ class SystemDatabase {
     }
     // ==================== Step Results ====================
     async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
         }
@@ -922,7 +989,7 @@ class SystemDatabase {
         return rows;
     }
     async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options);
         }
@@ -932,7 +999,7 @@ class SystemDatabase {
         }
     }
     async runTransactionalStep(workflowID, functionID, functionName, callback) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
@@ -1056,7 +1123,7 @@ class SystemDatabase {
         return await this.debounceDelayedWorkflowStandalone(params);
     }
     async debounceDelayedWorkflowStandalone(params) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const result = await this.#debounceDelayedWorkflowInternal(client, params);
@@ -1237,7 +1304,7 @@ class SystemDatabase {
         if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
             throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
         }
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Fetch the status of all original workflows inside the transaction.
@@ -1383,7 +1450,7 @@ class SystemDatabase {
             workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
         }
         const exportedWorkflows = [];
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             for (const wfID of workflowIDs) {
                 // Export workflow_status
@@ -1441,7 +1508,7 @@ class SystemDatabase {
         return exportedWorkflows;
     }
     async importWorkflow(workflows) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             for (const workflow of workflows) {
@@ -1597,6 +1664,10 @@ class SystemDatabase {
      * control-plane work hits the pool directly and bypasses the limiter.
      */
     #pollWithLimiter(query) {
+        // Closing our own pool used to end these waits; a caller's pool stays open, so end them here.
+        if (this.#destroyed) {
+            return Promise.reject(new error_1.DBOSError('The system database has been shut down'));
+        }
         return this.pollLimiter.runExclusive(query);
     }
     /**
@@ -1711,7 +1782,7 @@ class SystemDatabase {
     async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
         topic = topic ?? this.nullTopic;
         const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
@@ -1737,12 +1808,22 @@ class SystemDatabase {
             client.release();
         }
     }
-    async sendDirect(destinationID, message, topic, serialization, idempotencyKey) {
+    /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
+    async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
+        if (client !== undefined) {
+            return await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
+        }
+        return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
+    }
+    async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
+        return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
+    }
+    async #sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey) {
         topic = topic ?? this.nullTopic;
         // Same per-destination scoping as send() above.
         const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
         try {
-            await this.pool.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
+            await db.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
          VALUES ($1, $2, $3, $4, $5)
          ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
         }
@@ -1808,7 +1889,7 @@ class SystemDatabase {
         // Transactionally consume and return the message if it's in the DB, otherwise return null.
         let message = null;
         let serialization = null;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
             const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
@@ -1848,7 +1929,7 @@ class SystemDatabase {
     }
     // ==================== Events ====================
     async setEvent(workflowID, functionID, key, message, serialization) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Only a real write (not a replay) should wake readers.
@@ -1996,7 +2077,7 @@ class SystemDatabase {
     }
     // ==================== Streams ====================
     async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Find the maximum offset for this workflow_uuid and key combination
@@ -2022,7 +2103,7 @@ class SystemDatabase {
         }
     }
     async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Only a real insert (not a replay) should wake readers.
@@ -2142,7 +2223,7 @@ class SystemDatabase {
             }
             try {
                 // One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
-                const client = await this.pool.connect();
+                const client = await this.#connect();
                 try {
                     await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
                 }
@@ -2158,7 +2239,7 @@ class SystemDatabase {
     }
     // ==================== Observability: Workflow Communications ====================
     async getAllEvents(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
          WHERE workflow_uuid = $1`, [workflowID]);
@@ -2173,7 +2254,7 @@ class SystemDatabase {
         }
     }
     async getAllNotifications(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
          FROM "${this.schemaName}".notifications
@@ -2191,7 +2272,7 @@ class SystemDatabase {
         }
     }
     async getAllStreamEntries(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
          WHERE workflow_uuid = $1
@@ -2264,7 +2345,7 @@ class SystemDatabase {
             partitionFilter = `AND queue_partition_key = $PARTITION`;
             partitionParams.push(queuePartitionKey);
         }
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             // Default to READ COMMITTED except with global concurrency limits or rate limits
             if (queue.concurrency !== undefined || queue.rateLimit !== undefined) {
@@ -2419,7 +2500,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with concurrency 1 and no rate limit: ${queue.name}`);
         }
         // workerConcurrency needs no handling here: dispatch routes 0 (the pause-dequeue idiom) to the fallback path, and validation caps any other value at concurrency=1, which the PENDING gate already enforces globally.
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const latestVersion = await this.#latestApplicationVersionName(client);
@@ -3219,7 +3300,7 @@ class SystemDatabase {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
     }
     async applySchedules(schedules) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             for (const sched of schedules) {
@@ -3271,7 +3352,7 @@ class SystemDatabase {
      */
     async createApplicationVersion(versionName, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
@@ -3302,7 +3383,7 @@ class SystemDatabase {
      */
     async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -3408,7 +3489,7 @@ class SystemDatabase {
           application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
             : `ON CONFLICT (name) DO NOTHING`;
         const owner = record.applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
@@ -3527,7 +3608,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
         }
         // Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         let queues, schedules, versions, inFlight;
         try {
             await client.query('BEGIN');
@@ -3815,7 +3896,7 @@ class SystemDatabase {
         const startTimeMs = Date.now();
         // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
         const endTimeMs = startTimeMs + Math.ceil(durationMS);
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
             if (res) {
@@ -3847,7 +3928,7 @@ class SystemDatabase {
             this.notificationsClient = null;
         }
         client.removeAllListeners();
-        // Errors can still arrive while release() tears the connection down; a bare emit would crash the process.
+        // Cover the release() call itself, which tears the connection down and can surface a socket error.
         client.on('error', () => { });
         try {
             client.release(true);
@@ -3855,6 +3936,10 @@ class SystemDatabase {
         catch (e) {
             this.logger.warn(`Error releasing notifications client: ${String(e)}`);
         }
+        // release() re-attached pg's idle listener, which would forward this dead client's error on to the
+        // pool. A caller's pool may have no 'error' listener at all, so this client's death has to stay ours.
+        client.removeAllListeners('error');
+        client.on('error', (e) => this.logger.warn(`Error on retired notifications client: ${e}`));
     }
     // Shutdown can begin during any await in the setup below; releasing the client instead of carrying on
     // keeps pool.end() from waiting on a connection that will never be published.
@@ -3878,7 +3963,7 @@ class SystemDatabase {
             };
             let acquired = null;
             try {
-                const client = await this.pool.connect();
+                const client = await this.#connect();
                 acquired = client;
                 if (this.#abandonIfStopped(client))
                     return;
@@ -3960,7 +4045,7 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object, Object]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "initWorkflowStatus", null);
+], SystemDatabase.prototype, "initWorkflowStatusStandalone", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -4062,7 +4147,7 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [String, Object, Object, Object, String]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "sendDirect", null);
+], SystemDatabase.prototype, "sendDirectStandalone", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),