@dbos-inc/dbos-sdk 4.26.7-preview → 4.26.9-preview

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -100,49 +100,94 @@ async function grantDbosSchemaPermissions(databaseUrl, roleName, logger, schemaN
     }
 }
 exports.grantDbosSchemaPermissions = grantDbosSchemaPermissions;
-async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
-    let client = null;
+/**
+ * Check out a connection carrying our own error handler, so a socket death while we hold it is
+ * logged instead of thrown at an emitter with no listener. `release()` takes the handler back off,
+ * so nothing we attach outlives the checkout and the pool itself is never touched.
+ */
+async function borrowClient(pool, onError) {
+    const client = await pool.connect();
+    // Nothing awaits between the checkout and this attach, nor between the removal and release()
+    // below, so there is no turn of the event loop in which we hold the connection unguarded.
+    client.on('error', onError);
+    const release = client.release.bind(client);
+    client.release = (err) => {
+        client.removeListener('error', onError);
+        release(err);
+    };
+    return client;
+}
+/** Connect to the system database without creating it. The caller releases the client. */
+async function connectToSystemDatabase(sysDbUrl, logger, customPool) {
     if (customPool) {
-        // If a custom pool is passed in, assume the database already exists and create
-        // a client to run migrations.
-        client = await customPool.connect();
+        return await borrowClient(customPool, (err) => logger.warn(`Unexpected error in system database client: ${err}`));
     }
-    else {
-        // Otherwise, create the system database if it does not exist.
-        await (0, database_utils_1.ensurePGDatabase)(sysDbUrl, logger);
-        const sysClient = new pg_1.Client((0, utils_2.getClientConfig)(sysDbUrl));
-        // An 'error' event with no listener would take down the process.
-        sysClient.on('error', (err) => logger.warn(`Unexpected error in system database client: ${err}`));
-        try {
-            await sysClient.connect();
+    const sysClient = new pg_1.Client((0, utils_2.getClientConfig)(sysDbUrl));
+    // An 'error' event with no listener would take down the process.
+    sysClient.on('error', (err) => logger.warn(`Unexpected error in system database client: ${err}`));
+    try {
+        await sysClient.connect();
+    }
+    catch (e) {
+        await sysClient.end().catch(() => { });
+        throw new error_1.DBOSInitializationError(`Unable to connect to system database at ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)}: ${e.message}`, e instanceof Error ? e : undefined);
+    }
+    return sysClient;
+}
+async function releaseSystemDatabaseClient(client, customPool) {
+    try {
+        if (customPool) {
+            client.release();
         }
-        catch (e) {
-            await sysClient.end().catch(() => { });
-            throw new error_1.DBOSInitializationError(`Unable to connect to system database at ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)}: ${e.message}`, e instanceof Error ? e : undefined);
+        else {
+            await client.end();
         }
-        client = sysClient;
     }
+    catch (e) { }
+}
+async function isCockroachDB(client) {
+    const versionRes = await client.query('SELECT version() AS version');
+    return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
+}
+async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
+    if (!customPool) {
+        // A custom pool means the database already exists; otherwise, create it if it does not.
+        await (0, database_utils_1.ensurePGDatabase)(sysDbUrl, logger);
+    }
+    const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
     try {
-        const versionRes = await client.query('SELECT version() AS version');
-        const isCockroach = /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
+        const isCockroach = await isCockroachDB(client);
         await (0, migration_runner_1.runSysMigrationsPg)(client, (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }), schemaName, {
             onWarn: (e) => logger.info(e),
             isCockroach,
         });
     }
     finally {
-        try {
-            if (customPool) {
-                client.release();
-            }
-            else {
-                await client.end();
-            }
-        }
-        catch (e) { }
+        await releaseSystemDatabaseClient(client, customPool);
     }
 }
 exports.ensureSystemDatabase = ensureSystemDatabase;
+/** Check the system database is migrated to the version this build requires, creating and changing nothing. */
+async function verifySystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
+    const client = await connectToSystemDatabase(sysDbUrl, logger, customPool);
+    try {
+        const isCockroach = await isCockroachDB(client);
+        const requiredVersion = (0, migrations_1.allMigrations)(schemaName, { useListenNotify, isCockroach }).length;
+        const currentVersion = await (0, migration_runner_1.getCurrentSysDBVersion)(client, schemaName);
+        // A database ahead of this build belongs to a newer peer, which the migration runner also tolerates.
+        if (currentVersion < requiredVersion) {
+            throw new error_1.DBOSInitializationError(`System database ${(0, database_utils_1.maskDatabaseUrl)(sysDbUrl)} is at schema version ${currentVersion}, but this version ` +
+                `of DBOS requires ${requiredVersion}. This process is configured with runMigrations disabled, so it ` +
+                `will not migrate it: either migrate the system database out of band (\`npx dbos schema\`) or launch ` +
+                `with runMigrations enabled.`);
+        }
+        logger.debug(`System database schema version ${currentVersion} satisfies the required version ${requiredVersion}`);
+    }
+    finally {
+        await releaseSystemDatabaseClient(client, customPool);
+    }
+}
+exports.verifySystemDatabase = verifySystemDatabase;
 class NotificationMap {
     map = new Map();
     curCK = 0;
@@ -441,6 +486,8 @@ class SystemDatabase {
     runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
     // Per-partition-key created_at cursors: keep per-key queue order monotonic across batches
     #batchCreatedAtCursors = new Map();
+    // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
+    #destroyed = false;
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
     appName) {
@@ -469,14 +516,20 @@ class SystemDatabase {
         const effectivePoolSize = this.pool.options.max ?? sysDbPoolSize;
         const pollingLimit = pollingConcurrency ?? Math.max(1, Math.floor(effectivePoolSize / 2));
         this.pollLimiter = new utils_1.Semaphore(pollingLimit);
-        this.pool.on('error', (err) => {
-            this.logger.warn(`Unexpected error in pool: ${err}`);
-        });
-        this.pool.on('connect', (client) => {
-            client.on('error', (err) => {
-                this.logger.warn(`Unexpected error in idle client: ${err}`);
+        // Only ever attach listeners to a pool we own; a caller's pool is theirs to instrument. Idle
+        // connections are all this covers, since #connect guards the ones we are holding.
+        if (!this.customPool) {
+            this.pool.on('error', (err) => {
+                this.logger.warn(`Unexpected error in pool: ${err}`);
             });
-        });
+        }
+    }
+    #onClientError = (err) => {
+        this.logger.warn(`Unexpected error on a system database connection: ${err}`);
+    };
+    /** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
+    #connect() {
+        return borrowClient(this.pool, this.#onClientError);
     }
     getSerializer() {
         return this.serializer;
@@ -535,8 +588,10 @@ class SystemDatabase {
             `Either ${takeANewName}, or, if '${current}' was renamed to '${owner}', ` +
             `re-own its rows first with dbos rename-application`);
     }
-    async init() {
-        await ensureSystemDatabase(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
+    /** Migrates the system database, or, when `runMigrations` is false, verifies it is already migrated. */
+    async init(runMigrations = true) {
+        const migrateOrVerify = runMigrations ? ensureSystemDatabase : verifySystemDatabase;
+        await migrateOrVerify(this.systemDatabaseUrl, this.logger, this.customPool ? this.pool : undefined, this.schemaName, this.shouldUseDBNotifications);
         if (this.shouldUseDBNotifications) {
             await this.#listenForNotifications();
             // Push coalesced stream and event notifications off the write path.
@@ -547,6 +602,7 @@ class SystemDatabase {
     async destroy() {
         // Set synchronously, before any await, so no reconnect is scheduled or published after this point.
         this.#notificationsStopped = true;
+        this.#destroyed = true;
         if (this.reconnectTimeout) {
             clearTimeout(this.reconnectTimeout);
             this.reconnectTimeout = null;
@@ -561,66 +617,28 @@ class SystemDatabase {
         if (this.notificationsClient) {
             this.#retireNotificationsClient(this.notificationsClient);
         }
-        await this.pool.end();
+        // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
+        if (!this.customPool) {
+            await this.pool.end();
+        }
     }
     // ==================== Workflow Status ====================
-    async initWorkflowStatus(initStatus, ownerXid, options) {
-        const client = await this.pool.connect();
+    /** Runs on `client` if given, joining its transaction; otherwise in its own retried transaction. */
+    async initWorkflowStatus(initStatus, ownerXid, client) {
+        if (client !== undefined) {
+            return await this.#initWorkflowStatusInternal(client, initStatus, ownerXid);
+        }
+        return await this.initWorkflowStatusStandalone(initStatus, ownerXid);
+    }
+    async initWorkflowStatusStandalone(initStatus, ownerXid) {
+        const client = await this.#connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            // Moving from enqueued to pending asks to increment recovery attempts... rather than in the recovery process
-            //  where it moves from pending back to enqueued.
-            const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid, !!options?.isRecoveryRequest || !!options?.isDequeuedRequest);
-            if (resRow.name !== initStatus.workflowName) {
-                const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if (resRow.class_name !== initStatus.workflowClassName) {
-                const msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if ((resRow.config_name || '') !== (initStatus.workflowConfigName || '')) {
-                const msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
-                throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
-            }
-            else if ((resRow.queue_name ?? undefined) !== (initStatus.queueName ?? undefined)) {
-                // This is a warning because a different queue name is not necessarily an error.
-                this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
-            }
-            const status = resRow.status;
-            const deadlineEpochMS = resRow.workflow_deadline_epoch_ms ?? undefined;
-            // If there is an existing DB record and we aren't here to recover it,
-            //  leave it be.  Roll back the change to max recovery attempts.
-            if (ownerXid !== resRow.owner_xid && !options?.isRecoveryRequest && !options?.isDequeuedRequest) {
-                // It is not clear if getting the handle should throw the error, or getting the result from the handle should error.
-                //  Current precedent is the former.
-                if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
-                    throw new error_1.DBOSMaxRecoveryAttemptsExceededError(initStatus.workflowUUID, options?.maxRetries ?? -1);
-                }
-                return { status, deadlineEpochMS, shouldExecuteOnThisExecutor: false, serialization: resRow.serialization };
-            }
-            // Upsert above already set executor assignment and incremented the recovery attempt
-            shouldCommit = true;
-            // recovery_attempt means "attempts" (we kept the name for backward compatibility). It's default value is 1.
-            // Every time we init the status, we increment `recovery_attempts` by 1.
-            // Thus, when this number becomes equal to `maxRetries + 1`, we should mark the workflow as `MAX_RECOVERY_ATTEMPTS_EXCEEDED`.
-            const attempts = resRow.recovery_attempts;
-            if (options?.maxRetries && attempts > options?.maxRetries + 1) {
-                await this.updateWorkflowStatus(client, initStatus.workflowUUID, workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, {
-                    where: { status: workflow_1.StatusString.PENDING },
-                    throwOnFailure: false,
-                    update: { resetDeduplicationID: true },
-                });
-                throw new error_1.DBOSMaxRecoveryAttemptsExceededError(initStatus.workflowUUID, options.maxRetries);
-            }
-            this.logger.debug(`Workflow ${initStatus.workflowUUID} attempt number: ${attempts}.`);
-            return {
-                status,
-                deadlineEpochMS,
-                shouldExecuteOnThisExecutor: true,
-                serialization: resRow.serialization,
-            };
+            const result = await this.#initWorkflowStatusInternal(client, initStatus, ownerXid);
+            // If there is an existing DB record and we aren't here to recover it, leave it be.
+            shouldCommit = result.shouldExecuteOnThisExecutor;
+            return result;
         }
         finally {
             try {
@@ -637,6 +655,47 @@ class SystemDatabase {
             }
         }
     }
+    async #initWorkflowStatusInternal(client, initStatus, ownerXid) {
+        const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid);
+        if (resRow.name !== initStatus.workflowName) {
+            const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if (resRow.class_name !== initStatus.workflowClassName) {
+            const msg = `Workflow already exists with a different class name: ${resRow.class_name}, but the provided class name is: ${initStatus.workflowClassName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if ((resRow.config_name || '') !== (initStatus.workflowConfigName || '')) {
+            const msg = `Workflow already exists with a different class configuration: ${resRow.config_name}, but the provided class configuration is: ${initStatus.workflowConfigName}`;
+            throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
+        }
+        else if ((resRow.queue_name ?? undefined) !== (initStatus.queueName ?? undefined)) {
+            // This is a warning because a different queue name is not necessarily an error.
+            this.logger.warn(`Workflow (${initStatus.workflowUUID}) already exists in queue: ${resRow.queue_name}, but the provided queue name is: ${initStatus.queueName}. The queue is not updated. ${new Error().stack}`);
+        }
+        const status = resRow.status;
+        const deadlineEpochMS = resRow.workflow_deadline_epoch_ms ?? undefined;
+        // The upsert above already set executor assignment for a row we own.
+        return {
+            status,
+            deadlineEpochMS,
+            shouldExecuteOnThisExecutor: ownerXid === resRow.owner_xid,
+            serialization: resRow.serialization,
+        };
+    }
+    /** Move claimed workflows that exhausted their attempts off the queue, leaving rows others have moved on alone. */
+    async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
+        if (workflowIDs.length === 0)
+            return;
+        await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
+       SET status = $1,
+           deduplication_id = NULL,
+           started_at_epoch_ms = NULL,
+           queue_name = NULL,
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
+           completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = ANY($2::text[]) AND status = $3 AND recovery_attempts >= $4`, [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflowIDs, workflow_1.StatusString.PENDING, minRecoveryAttempts]);
+    }
     /** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
     async #maxPartitionKeyCreatedAt(keys) {
         const maxima = new Map();
@@ -745,7 +804,7 @@ class SystemDatabase {
             'schedule_name',
             'application_name',
         ];
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Chunk to stay well under the bind-parameter limit.
@@ -783,7 +842,7 @@ class SystemDatabase {
         return inserted;
     }
     async recordWorkflowOutput(workflowID, status) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output });
         }
@@ -792,7 +851,7 @@ class SystemDatabase {
         }
     }
     async recordWorkflowError(workflowID, status) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error });
         }
@@ -832,24 +891,17 @@ class SystemDatabase {
     }
     // Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
     async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
-        const params = [
-            workflow_1.StatusString.ENQUEUED,
-            Date.now(),
-            recoveryQueueName,
-            workflow_1.StatusString.PENDING,
-            executorID,
-            appVersion,
-        ];
+        const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
         // executor_id defaults to "local", so it collides across applications.
         const scope = this.#appNameFilter('application_name', this.appName, params);
         const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET started_at_epoch_ms = NULL,
            status = $1,
-           updated_at = $2,
-           queue_name = COALESCE(queue_name, $3)
-       WHERE status = $4
-         AND executor_id = $5
-         AND application_version = $6
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
+           queue_name = COALESCE(queue_name, $2)
+       WHERE status = $3
+         AND executor_id = $4
+         AND application_version = $5
          AND ${scope}
        RETURNING workflow_uuid`, params);
         return result.rows.map((row) => row.workflow_uuid);
@@ -861,7 +913,7 @@ class SystemDatabase {
             return status ? JSON.stringify(status) : null;
         };
         if (callerID && callerFN) {
-            const client = await this.pool.connect();
+            const client = await this.#connect();
             try {
                 // Check if the operation has been done before for OAOO (only do this inside a workflow).
                 const json = await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus);
@@ -879,12 +931,33 @@ class SystemDatabase {
             return json ? JSON.parse(json) : null;
         }
     }
+    /** Max IDs per {@link getWorkflowStatuses} fetch: listWorkflows binds one parameter per ID. */
+    statusFetchChunkSize = 500;
+    // Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
+    async fetchWorkflowStatusChunk(workflowIDs) {
+        return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
+    }
+    /** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
+    async getWorkflowStatuses(workflowIDs) {
+        const statuses = new Map();
+        for (let start = 0; start < workflowIDs.length; start += this.statusFetchChunkSize) {
+            for (const status of await this.fetchWorkflowStatusChunk(workflowIDs.slice(start, start + this.statusFetchChunkSize))) {
+                statuses.set(status.workflowUUID, status);
+            }
+        }
+        return statuses;
+    }
     // Only used in tests
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await this.updateWorkflowStatus(client, workflowID, status, {
-                update: { resetRecoveryAttempts, resetNameTo: internalOptions?.updateName },
+                update: {
+                    resetRecoveryAttempts,
+                    resetNameTo: internalOptions?.updateName,
+                    queueName: internalOptions?.queueName,
+                    resetStartedAtEpochMs: internalOptions?.resetStartedAtEpochMs,
+                },
             });
         }
         finally {
@@ -893,7 +966,7 @@ class SystemDatabase {
     }
     // ==================== Step Results ====================
     async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
         }
@@ -916,7 +989,7 @@ class SystemDatabase {
         return rows;
     }
     async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options);
         }
@@ -926,7 +999,7 @@ class SystemDatabase {
         }
     }
     async runTransactionalStep(workflowID, functionID, functionName, callback) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
@@ -1023,15 +1096,15 @@ class SystemDatabase {
     }
     async setWorkflowPriority(workflowID, priority) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET priority = $1, updated_at = $2
-       WHERE workflow_uuid = $3
-         AND status IN ($4, $5)`, [priority, Date.now(), workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
+       SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = $2
+         AND status IN ($3, $4)`, [priority, workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
     }
     async setWorkflowDelay(workflowID, delayUntilEpochMS) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET delay_until_epoch_ms = $1, updated_at = $2
-       WHERE workflow_uuid = $3
-         AND status = $4`, [delayUntilEpochMS, Date.now(), workflowID, workflow_1.StatusString.DELAYED]);
+       SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = $2
+         AND status = $3`, [delayUntilEpochMS, workflowID, workflow_1.StatusString.DELAYED]);
     }
     /**
      * Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
@@ -1050,7 +1123,7 @@ class SystemDatabase {
         return await this.debounceDelayedWorkflowStandalone(params);
     }
     async debounceDelayedWorkflowStandalone(params) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const result = await this.#debounceDelayedWorkflowInternal(client, params);
@@ -1071,7 +1144,6 @@ class SystemDatabase {
             params.delayUntilEpochMS,
             params.input,
             params.serialization,
-            Date.now(),
             params.workflowName,
             classNameOrNull,
             params.queueName,
@@ -1087,12 +1159,13 @@ class SystemDatabase {
              THEN debounce_deadline_epoch_ms
              ELSE $1
            END,
-           inputs = $2, serialization = $3, updated_at = $4,
+           inputs = $2, serialization = $3,
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            -- Claim it for the target, as its dequeue would: left unclaimed, every peer coalesces onto the one workflow and the last inputs win.
-           application_name = COALESCE(application_name, $10)
-       WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
-         AND queue_name = $7 AND deduplication_id = $8
-         AND status = $9 AND is_debounced = TRUE
+           application_name = COALESCE(application_name, $9)
+       WHERE name = $4 AND class_name IS NOT DISTINCT FROM $5
+         AND queue_name = $6 AND deduplication_id = $7
+         AND status = $8 AND is_debounced = TRUE
          AND ${ownScope}
        RETURNING workflow_uuid`, updateParams);
         if (updated.rows.length > 0) {
@@ -1231,7 +1304,7 @@ class SystemDatabase {
         if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
             throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
         }
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Fetch the status of all original workflows inside the transaction.
@@ -1377,7 +1450,7 @@ class SystemDatabase {
             workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
         }
         const exportedWorkflows = [];
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             for (const wfID of workflowIDs) {
                 // Export workflow_status
@@ -1435,7 +1508,7 @@ class SystemDatabase {
         return exportedWorkflows;
     }
     async importWorkflow(workflows) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             for (const workflow of workflows) {
@@ -1591,6 +1664,10 @@ class SystemDatabase {
      * control-plane work hits the pool directly and bypasses the limiter.
      */
     #pollWithLimiter(query) {
+        // Closing our own pool used to end these waits; a caller's pool stays open, so end them here.
+        if (this.#destroyed) {
+            return Promise.reject(new error_1.DBOSError('The system database has been shut down'));
+        }
         return this.pollLimiter.runExclusive(query);
     }
     /**
@@ -1705,7 +1782,7 @@ class SystemDatabase {
     async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
         topic = topic ?? this.nullTopic;
         const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
@@ -1731,12 +1808,22 @@ class SystemDatabase {
             client.release();
         }
     }
-    async sendDirect(destinationID, message, topic, serialization, idempotencyKey) {
+    /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
+    async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
+        if (client !== undefined) {
+            return await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
+        }
+        return await this.sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey);
+    }
+    async sendDirectStandalone(destinationID, message, topic, serialization, idempotencyKey) {
+        return await this.#sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey);
+    }
+    async #sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey) {
         topic = topic ?? this.nullTopic;
         // Same per-destination scoping as send() above.
         const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
         try {
-            await this.pool.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
+            await db.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
          VALUES ($1, $2, $3, $4, $5)
          ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
         }
@@ -1802,7 +1889,7 @@ class SystemDatabase {
         // Transactionally consume and return the message if it's in the DB, otherwise return null.
         let message = null;
         let serialization = null;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
             const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
@@ -1842,7 +1929,7 @@ class SystemDatabase {
     }
     // ==================== Events ====================
     async setEvent(workflowID, functionID, key, message, serialization) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Only a real write (not a replay) should wake readers.
@@ -1990,7 +2077,7 @@ class SystemDatabase {
     }
     // ==================== Streams ====================
     async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Find the maximum offset for this workflow_uuid and key combination
@@ -2016,7 +2103,7 @@ class SystemDatabase {
         }
     }
     async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Only a real insert (not a replay) should wake readers.
@@ -2136,7 +2223,7 @@ class SystemDatabase {
             }
             try {
                 // One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
-                const client = await this.pool.connect();
+                const client = await this.#connect();
                 try {
                     await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
                 }
@@ -2152,7 +2239,7 @@ class SystemDatabase {
     }
     // ==================== Observability: Workflow Communications ====================
     async getAllEvents(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
          WHERE workflow_uuid = $1`, [workflowID]);
@@ -2167,7 +2254,7 @@ class SystemDatabase {
         }
     }
     async getAllNotifications(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
          FROM "${this.schemaName}".notifications
@@ -2185,7 +2272,7 @@ class SystemDatabase {
         }
     }
     async getAllStreamEntries(workflowID) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
          WHERE workflow_uuid = $1
@@ -2216,7 +2303,7 @@ class SystemDatabase {
         // Only what this application would dequeue: a peer's debounce key is not ours to clear.
         const scope = this.#appNameFilter('application_name', this.appName, params);
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET status = $1, updated_at = $2,
+       SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
        WHERE status = $3 AND delay_until_epoch_ms <= $2 AND ${scope}`, params);
     }
@@ -2248,7 +2335,6 @@ class SystemDatabase {
         return rows.map((row) => row.pk);
     }
     async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey) {
-        const startTimeMs = Date.now();
         const limiterPeriodMS = queue.rateLimit ? queue.rateLimit.periodSec * 1000 : 0;
         const claimedIDs = [];
         const localRunningForQueue = this.countRunningWorkflowsForQueue(queue.name, queuePartitionKey);
@@ -2259,7 +2345,7 @@ class SystemDatabase {
             partitionFilter = `AND queue_partition_key = $PARTITION`;
             partitionParams.push(queuePartitionKey);
         }
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             // Default to READ COMMITTED except with global concurrency limits or rate limits
             if (queue.concurrency !== undefined || queue.rateLimit !== undefined) {
@@ -2275,7 +2361,7 @@ class SystemDatabase {
                     queue.name,
                     workflow_1.StatusString.ENQUEUED,
                     workflow_1.StatusString.DELAYED,
-                    startTimeMs - limiterPeriodMS,
+                    limiterPeriodMS,
                     ...partitionParams,
                 ];
                 // Count only what this application would dequeue, matching the select below.
@@ -2284,7 +2370,8 @@ class SystemDatabase {
            WHERE queue_name = $1
              AND rate_limited = TRUE
              AND status NOT IN ($2, $3)
-             AND started_at_epoch_ms > $4
+             -- Database clock on both sides, as the claim stamps started_at_epoch_ms with it.
+             AND started_at_epoch_ms > (EXTRACT(epoch FROM now()) * 1000)::bigint - $4
              AND ${scope}
              ${partitionFilter.replace('$PARTITION', '$5')}`, params);
                 numRecentQueries = Number(countResult.rows[0].count);
@@ -2297,9 +2384,13 @@ class SystemDatabase {
             // If there is a global or local concurrency limit N, select only the N oldest enqueued
             // functions, else select all of them.
             let maxTasks = Infinity;
+            if (queue.rateLimit) {
+                // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
+                maxTasks = Math.max(0, queue.rateLimit.limitPerPeriod - numRecentQueries);
+            }
             if (queue.workerConcurrency !== undefined) {
                 // Use the in-memory registry for this worker's running count — avoids a DB round trip.
-                maxTasks = Math.max(0, queue.workerConcurrency - localRunningForQueue);
+                maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningForQueue));
             }
             if (queue.concurrency !== undefined) {
                 // Global concurrency still requires a DB query since other workers may be running workflows too.
@@ -2327,7 +2418,10 @@ class SystemDatabase {
             const versionClause = isLatestVersion
                 ? '(application_version = $3 OR application_version IS NULL)'
                 : 'application_version = $3';
-            const lockMode = queue.concurrency ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
+            // A limit shared across processes needs a consistent view of the table: NOWAIT makes an
+            // overlapping dequeuer abort rather than claim the next rows and spend the same budget twice.
+            const sharedBudget = queue.concurrency !== undefined || queue.rateLimit !== undefined;
+            const lockMode = sharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
             const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
             const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
             const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
@@ -2349,43 +2443,41 @@ class SystemDatabase {
             await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_FIND_AND_MARK_AFTER_SELECT);
             // Start the workflows
             const workflowIDs = rows.map((row) => row.workflow_uuid);
-            for (const id of workflowIDs) {
-                // If we have a rate limit, stop starting functions when the number
-                //   of functions started this period exceeds the limit.
-                if (queue.rateLimit && claimedIDs.length + numRecentQueries >= queue.rateLimit.limitPerPeriod) {
-                    break;
-                }
+            if (workflowIDs.length > 0) {
                 // Start the functions by marking them as pending and updating their executor IDs.
-                // Only claim the workflow if the UPDATE actually transitioned an ENQUEUED row —
-                // otherwise another worker won the race and we must not re-dispatch it.
                 const updateParams = [
                     workflow_1.StatusString.PENDING,
                     executorID,
                     appVersion,
-                    startTimeMs,
                     queue.rateLimit !== undefined,
-                    id,
+                    workflowIDs,
                     workflow_1.StatusString.ENQUEUED,
                     // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
                     this.appName ?? null,
                 ];
                 // Re-check ownership alongside status, as the partitioned claim guard does.
                 const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
-                const updateRes = await client.query(`UPDATE "${this.schemaName}".workflow_status
+                // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
+                const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
            SET status = $1,
                executor_id = $2,
                application_version = $3,
-               started_at_epoch_ms = $4,
-               rate_limited = $5,
-               application_name = COALESCE(application_name, $8),
+               started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
+               rate_limited = $4,
+               application_name = COALESCE(application_name, $7),
+               recovery_attempts = recovery_attempts + 1,
+               updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
                workflow_deadline_epoch_ms = CASE
                  WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
                  THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
                  ELSE workflow_deadline_epoch_ms
                END
-           WHERE workflow_uuid = $6 AND status = $7 AND ${claimScope}`, updateParams);
-                if ((updateRes.rowCount ?? 0) > 0) {
-                    claimedIDs.push(id);
+           WHERE workflow_uuid = ANY($5::text[]) AND status = $6 AND ${claimScope}
+           RETURNING workflow_uuid`, updateParams);
+                const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
+                for (const id of workflowIDs) {
+                    if (flippedIDs.has(id))
+                        claimedIDs.push(id);
                 }
             }
             await client.query('COMMIT');
@@ -2408,8 +2500,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with concurrency 1 and no rate limit: ${queue.name}`);
         }
         // workerConcurrency needs no handling here: dispatch routes 0 (the pause-dequeue idiom) to the fallback path, and validation caps any other value at concurrency=1, which the PENDING gate already enforces globally.
-        const startTimeMs = Date.now();
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const latestVersion = await this.#latestApplicationVersionName(client);
@@ -2495,7 +2586,6 @@ class SystemDatabase {
                 workflow_1.StatusString.ENQUEUED,
                 queue.name,
                 appVersion,
-                startTimeMs,
                 // Claim the row, as the unpartitioned dequeue does.
                 this.appName ?? null,
             ];
@@ -2505,9 +2595,11 @@ class SystemDatabase {
          SET status = $1,
              executor_id = $2,
              application_version = $6,
-             started_at_epoch_ms = $7,
+             started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
              rate_limited = FALSE,
-             application_name = COALESCE(application_name, $8),
+             application_name = COALESCE(application_name, $7),
+             recovery_attempts = recovery_attempts + 1,
+             updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
              workflow_deadline_epoch_ms = CASE
                WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
                THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
@@ -3208,7 +3300,7 @@ class SystemDatabase {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
     }
     async applySchedules(schedules) {
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             for (const sched of schedules) {
@@ -3260,7 +3352,7 @@ class SystemDatabase {
      */
     async createApplicationVersion(versionName, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
@@ -3291,7 +3383,7 @@ class SystemDatabase {
      */
     async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -3397,7 +3489,7 @@ class SystemDatabase {
           application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
             : `ON CONFLICT (name) DO NOTHING`;
         const owner = record.applicationName ?? this.appName;
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             await client.query('BEGIN');
             const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
@@ -3516,7 +3608,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
         }
         // Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         let queues, schedules, versions, inFlight;
         try {
             await client.query('BEGIN');
@@ -3550,7 +3642,7 @@ class SystemDatabase {
         return { queues, schedules, versions, workflows: inFlight + terminal, steps };
     }
     // ==================== Internal ====================
-    async insertWorkflowStatus(client, initStatus, ownerXid, incrementAttempts = false) {
+    async insertWorkflowStatus(client, initStatus, ownerXid) {
         try {
             const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
           workflow_uuid,
@@ -3566,9 +3658,7 @@ class SystemDatabase {
           executor_id,
           application_version,
           application_id,
-          created_at,
           recovery_attempts,
-          updated_at,
           workflow_timeout_ms,
           workflow_deadline_epoch_ms,
           inputs,
@@ -3585,21 +3675,16 @@ class SystemDatabase {
           debounce_deadline_epoch_ms,
           is_debounced,
           application_name
-        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30, $31, $32, $33)
+        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
         ON CONFLICT (workflow_uuid)
           DO UPDATE SET
-            recovery_attempts = CASE
-              WHEN workflow_status.status != '${workflow_1.StatusString.ENQUEUED}' AND workflow_status.status != '${workflow_1.StatusString.DELAYED}'
-              THEN workflow_status.recovery_attempts + $25
-              ELSE workflow_status.recovery_attempts
-            END,
-            updated_at = EXCLUDED.updated_at,
+            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
             executor_id = CASE
               WHEN EXCLUDED.status != '${workflow_1.StatusString.ENQUEUED}' AND EXCLUDED.status != '${workflow_1.StatusString.DELAYED}'
               THEN EXCLUDED.executor_id
               ELSE workflow_status.executor_id
             END
-          RETURNING recovery_attempts, status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
+          RETURNING status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
                 initStatus.workflowUUID,
                 initStatus.status,
                 initStatus.workflowName,
@@ -3614,9 +3699,7 @@ class SystemDatabase {
                 initStatus.executorId,
                 initStatus.applicationVersion ?? null,
                 initStatus.applicationID,
-                initStatus.createdAt,
                 initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
-                initStatus.updatedAt ?? Date.now(),
                 initStatus.timeoutMS ?? null,
                 initStatus.deadlineEpochMS ?? null,
                 initStatus.input ?? null,
@@ -3625,7 +3708,6 @@ class SystemDatabase {
                 initStatus.queuePartitionKey ?? null,
                 initStatus.forkedFrom ?? null,
                 initStatus.parentWorkflowID ?? null,
-                (incrementAttempts ?? false) ? 1 : 0,
                 initStatus.serialization,
                 ownerXid,
                 initStatus.delayUntilEpochMS ?? null,
@@ -3814,7 +3896,7 @@ class SystemDatabase {
         const startTimeMs = Date.now();
         // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
         const endTimeMs = startTimeMs + Math.ceil(durationMS);
-        const client = await this.pool.connect();
+        const client = await this.#connect();
         try {
             const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
             if (res) {
@@ -3846,7 +3928,7 @@ class SystemDatabase {
             this.notificationsClient = null;
         }
         client.removeAllListeners();
-        // Errors can still arrive while release() tears the connection down; a bare emit would crash the process.
+        // Cover the release() call itself, which tears the connection down and can surface a socket error.
         client.on('error', () => { });
         try {
             client.release(true);
@@ -3854,6 +3936,10 @@ class SystemDatabase {
         catch (e) {
             this.logger.warn(`Error releasing notifications client: ${String(e)}`);
         }
+        // release() re-attached pg's idle listener, which would forward this dead client's error on to the
+        // pool. A caller's pool may have no 'error' listener at all, so this client's death has to stay ours.
+        client.removeAllListeners('error');
+        client.on('error', (e) => this.logger.warn(`Error on retired notifications client: ${e}`));
     }
     // Shutdown can begin during any await in the setup below; releasing the client instead of carrying on
     // keeps pool.end() from waiting on a connection that will never be published.
@@ -3877,7 +3963,7 @@ class SystemDatabase {
             };
             let acquired = null;
             try {
-                const client = await this.pool.connect();
+                const client = await this.#connect();
                 acquired = client;
                 if (this.#abandonIfStopped(client))
                     return;
@@ -3957,9 +4043,15 @@ exports.SystemDatabase = SystemDatabase;
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Object, Object, Object]),
+    __metadata("design:paramtypes", [Object, Object]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "initWorkflowStatus", null);
+], SystemDatabase.prototype, "initWorkflowStatusStandalone", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, Number]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "deadLetterWorkflows", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -3978,6 +4070,12 @@ __decorate([
     __metadata("design:paramtypes", [String, String, Number]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "getWorkflowStatus", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "fetchWorkflowStatusChunk", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -4049,7 +4147,7 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [String, Object, Object, Object, String]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "sendDirect", null);
+], SystemDatabase.prototype, "sendDirectStandalone", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),