@dbos-inc/dbos-sdk 5.2.10-preview → 5.3.0

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.retriablePostgresException = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.advisoryLockKey = exports.retentionLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -88,8 +88,6 @@ function setsIdleTransactionTimeout(databaseUrl) {
 }
 // Workflows re-owned per transaction by a rename.
 exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
-// Rows deleted per transaction by garbage collection.
-exports.DEFAULT_GC_BATCH_SIZE = 50_000;
 const QUEUE_COLUMN_BY_FIELD = {
     concurrency: 'concurrency',
     workerConcurrency: 'worker_concurrency',
@@ -204,16 +202,6 @@ async function releaseSystemDatabaseClient(client, customPool) {
     }
     catch (e) { }
 }
-/**
- * The pg_try_advisory_lock argument for one schema's retention round: the leading 8 bytes of
- * SHA-256 over a fixed string, read as a signed big-endian 64-bit integer. Separate locks for
- * separate schemas. Every DBOS SDK derives the key this way, so rounds in different languages
- * against one system database contend for the same lock; changing it here changes it everywhere.
- */
-function retentionLockKey(schemaName) {
-    return advisoryLockKey(`dbos.retention.${schemaName}`);
-}
-exports.retentionLockKey = retentionLockKey;
 /** A Postgres advisory lock key: the leading 8 bytes of SHA-256 over `lockName`, as a signed big-endian integer. */
 function advisoryLockKey(lockName) {
     return (0, crypto_1.createHash)('sha256').update(lockName).digest().readBigInt64BE(0);
@@ -223,6 +211,7 @@ async function isCockroachDB(client) {
     const versionRes = await client.query('SELECT version() AS version');
     return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
 }
+exports.isCockroachDB = isCockroachDB;
 async function ensureSystemDatabase(sysDbUrl, logger, customPool, schemaName = 'dbos', useListenNotify = true) {
     if (!customPool) {
         // A custom pool means the database already exists; otherwise, create it if it does not.
@@ -378,14 +367,6 @@ const RETRY_SQLSTATE_CODES = new Set([
     '25P03', // idle_in_transaction_session_timeout, when the kill lands on an in-flight query; an idle kill surfaces as a message match below
     '40P01', // deadlock_detected: the victim's transaction rolled back, so rerunning it is safe
 ]);
-/**
- * Bulk maintenance retries these a bounded number of times. 40001 is kept out of the sets above,
- * which feed `dbRetry` and so retry forever; every other path runs READ COMMITTED and never sees it.
- */
-const SERIALIZATION_SQLSTATE_CODES = new Set([
-    '40001', // serialization_failure (MVCC conflict)
-    '40P01', // deadlock_detected
-]);
 // Node.js transient network error codes (system call level)
 const RETRY_NODE_ERRNOS = new Set([
     'ECONNRESET',
@@ -394,16 +375,21 @@ const RETRY_NODE_ERRNOS = new Set([
     'ENETUNREACH',
     'ETIMEDOUT',
     'ECONNABORTED',
+    'EAI_AGAIN', // DNS lookup failed temporarily, e.g. while a container's hostname is briefly unresolvable
 ]);
 function isPgDatabaseError(e) {
     // Matched by shape, not instanceof: a user-supplied pool may throw another pg copy's DatabaseError.
     return !!e && typeof e === 'object' && typeof e.code === 'string' && e.code.length === 5;
 }
+exports.isPgDatabaseError = isPgDatabaseError;
 function sqlStateLooksRetryable(sqlstate) {
     if (!sqlstate)
         return false;
     if (RETRY_SQLSTATE_CODES.has(sqlstate))
         return true;
+    // query_canceled (a statement timeout or an explicit cancel); retrying would undo it
+    if (sqlstate === '57014')
+        return false;
     const prefix = sqlstate.toString().slice(0, 2);
     return RETRY_SQLSTATE_PREFIXES.has(prefix);
 }
@@ -448,6 +434,7 @@ function* unwrapErrors(e) {
         yield cur;
     }
 }
+exports.unwrapErrors = unwrapErrors;
 // "What could possibly go wrong?"
 function retriablePostgresException(err) {
     // Dig into AggregateErrors of various types
@@ -480,80 +467,59 @@ exports.retriablePostgresException = retriablePostgresException;
 function isStatementTimeout(err) {
     return !!err && typeof err === 'object' && err.code === '57014';
 }
-function isSerializationError(err) {
-    for (const e of unwrapErrors(err)) {
-        const anyErr = e;
-        if (isPgDatabaseError(anyErr) && !!anyErr.code && SERIALIZATION_SQLSTATE_CODES.has(anyErr.code)) {
-            return true;
-        }
-    }
-    return false;
-}
-/**
- * Re-run a batch that lost a deadlock or serialization race. The database already rolled it
- * back, so replaying it is safe.
- */
-async function retryOnSerializationError(operation) {
-    const maxAttempts = 10;
-    const maxBackoff = 2.0;
-    let backoff = 0.05;
-    for (let attempt = 1;; attempt++) {
+/** Run `operation` under the {@link dbRetry} policy, for code that cannot take the decorator. */
+async function withDbRetry(operation, options = {}) {
+    // Read the defaults per call so the backoff stays tunable after the decorator is applied.
+    const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
+    let retries = 0;
+    let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
+    const abortSignal = (0, context_1.currentDbRetryAbortSignal)();
+    while (true) {
         try {
             return await operation();
         }
         catch (e) {
-            if (!isSerializationError(e)) {
-                throw e;
+            if (retriablePostgresException(e)) {
+                if (abortSignal?.aborted)
+                    throw e;
+                retries++;
+                // Calculate backoff with jitter
+                const actualBackoff = backoff * (0.5 + Math.random());
+                dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
+                    `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
+                // Sleep with backoff
+                if (abortSignal) {
+                    await (0, utils_1.interruptibleSleep)(actualBackoff * 1000, abortSignal);
+                    if (abortSignal.aborted)
+                        throw e;
+                }
+                else {
+                    await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
+                }
+                // Increase backoff for next attempt (exponential)
+                backoff = Math.min(backoff * 2, maxBackoff);
             }
-            const message = e instanceof Error ? e.message : String(e);
-            if (attempt === maxAttempts) {
-                dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Garbage collection failed after ${maxAttempts} attempts: ${message}`);
+            else {
                 throw e;
             }
-            // Jittered backoff, so peers that collided do not collide again
-            const actualBackoff = backoff * (0.5 + Math.random());
-            dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Contention or deadlock detected in workflow garbage collection: ${message}. ` +
-                `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${attempt})`);
-            await (0, utils_1.sleepms)(actualBackoff * 1000);
-            backoff = Math.min(backoff * 2, maxBackoff);
         }
     }
 }
+exports.withDbRetry = withDbRetry;
 /**
  * If a workflow encounters a database connection issue while performing an operation,
  * block the workflow and retry the operation until it reconnects and succeeds.
  * In other words, if DBOS loses its database connection, everything pauses until the connection is recovered,
  * trading off availability for correctness.
+ *
+ * The exception is a background DBOS loop run under `runWithAbortableDbRetries`: once its stop signal
+ * aborts, the last connection error is rethrown, so stopping the loop (e.g. at shutdown) cannot hang.
  */
 function dbRetry(options = {}) {
     return function (target, propertyName, descriptor) {
         const method = descriptor.value;
         descriptor.value = async function (...args) {
-            // Read the defaults per call so the backoff stays tunable after the decorator is applied.
-            const maxBackoff = options.maxBackoff ?? utils_1.dbRetryConfig.maxBackoffSec;
-            let retries = 0;
-            let backoff = options.initialBackoff ?? utils_1.dbRetryConfig.initialBackoffSec;
-            while (true) {
-                try {
-                    return await method.apply(this, args);
-                }
-                catch (e) {
-                    if (retriablePostgresException(e)) {
-                        retries++;
-                        // Calculate backoff with jitter
-                        const actualBackoff = backoff * (0.5 + Math.random());
-                        dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Database connection failed: ${e instanceof Error ? e.message : String(e)}. ` +
-                            `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${retries})`);
-                        // Sleep with backoff
-                        await (0, utils_1.sleepms)(actualBackoff * 1000); // Convert to milliseconds
-                        // Increase backoff for next attempt (exponential)
-                        backoff = Math.min(backoff * 2, maxBackoff);
-                    }
-                    else {
-                        throw e;
-                    }
-                }
-            }
+            return withDbRetry(() => method.apply(this, args), options);
         };
         return descriptor;
     };
@@ -635,11 +601,12 @@ class SystemDatabase {
     #batchCreatedAtCursors = new Map();
     // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
     #destroyed = false;
-    // Resolved on first use by #cockroach(), since detecting it costs a query.
-    #isCockroach = undefined;
+    get destroyed() {
+        return this.#destroyed;
+    }
     // Connections a retention round holds right now. destroy() cuts them, since closing the
     // pool would otherwise wait on the lock session and on any statement in flight.
-    #retentionClients = new Set();
+    retentionClients = new Set();
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
     appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS, idleTransactionTimeoutMs = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS) {
@@ -699,25 +666,9 @@ class SystemDatabase {
         this.logger.warn(`Unexpected error on a system database connection: ${err}`);
     };
     /** Check out a pool connection guarded for as long as we hold it. See {@link borrowClient}. */
-    #connect() {
+    connect() {
         return borrowClient(this.pool, this.#onClientError);
     }
-    /** Borrow a connection for a retention round, so destroy() can cut it. */
-    async #borrowRetentionClient() {
-        if (this.#destroyed) {
-            throw new Error('System database shutting down');
-        }
-        const client = await this.#connect();
-        this.#retentionClients.add(client);
-        return client;
-    }
-    /** Return a retention connection, unless destroy() already cut it: a second release would throw. */
-    #releaseRetentionClient(client) {
-        if (this.#retentionClients.delete(client)) {
-            // No error argument: a genuinely dead connection is still evicted by the pool's own check.
-            client.release();
-        }
-    }
     /**
      * Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
      * snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
@@ -728,7 +679,7 @@ class SystemDatabase {
      */
     async observabilityQuery(fn) {
         const timeoutMs = this.observabilityQueryTimeoutMs;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             if (timeoutMs === undefined) {
                 return await fn(client);
@@ -744,7 +695,7 @@ class SystemDatabase {
             }
             catch (e) {
                 await client.query('ROLLBACK').catch(() => { });
-                // No `cause`: dbRetry walks the cause chain and would retry 57014 forever as an operator intervention.
+                // No `cause`: the typed timeout replaces the driver's query_canceled error.
                 throw isStatementTimeout(e) ? new error_1.DBOSQueryTimeoutError(timeoutMs) : e;
             }
         }
@@ -757,7 +708,7 @@ class SystemDatabase {
      * A predicate matching rows owned by these applications plus unclaimed ones, which belong to
      * every application; unset or empty matches everything. Appends its bind parameter to `params`.
      */
-    #appNameFilter(column, value, params) {
+    appNameFilter(column, value, params) {
         // An empty name is no name: it is not a value any application could be configured with.
         const names = !value ? [] : Array.isArray(value) ? value : [value];
         if (names.length === 0)
@@ -770,8 +721,8 @@ class SystemDatabase {
      * this application owns, not to every application's rows. A handle with no application of its
      * own still matches every one.
      */
-    #observabilityFilter(column, value, params) {
-        return this.#appNameFilter(column, value ?? this.appName, params);
+    observabilityFilter(column, value, params) {
+        return this.appNameFilter(column, value ?? this.appName, params);
     }
     /**
      * The version name a dequeue treats as latest: this application's own plus unclaimed
@@ -779,7 +730,7 @@ class SystemDatabase {
      */
     async #latestApplicationVersionName(client) {
         const params = [];
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         const { rows } = await client.query(`SELECT version_name
          FROM "${this.schemaName}".application_versions
         WHERE ${scope}
@@ -842,7 +793,7 @@ class SystemDatabase {
         // A retention round still running is cut here rather than waited for: returned with an
         // error, each connection is destroyed at once, the idle lock session lets go of the
         // advisory lock, and the round fails on its next statement instead of holding shutdown.
-        for (const client of this.#retentionClients) {
+        for (const client of this.retentionClients) {
             // Cover the release() call itself, which tears the connection down and can surface a socket error.
             client.on('error', () => { });
             try {
@@ -852,7 +803,7 @@ class SystemDatabase {
                 this.logger.warn(`Error releasing a retention connection: ${String(e)}`);
             }
         }
-        this.#retentionClients.clear();
+        this.retentionClients.clear();
         // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
         if (!this.customPool) {
             await this.pool.end();
@@ -867,7 +818,7 @@ class SystemDatabase {
         return await this.initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy);
     }
     async initWorkflowStatusStandalone(initStatus, creatorXid, reusePolicy) {
-        const client = await this.#connect();
+        const client = await this.connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -893,7 +844,7 @@ class SystemDatabase {
     }
     /** Insert a child workflow's row and record it as the parent's step in one transaction, so a crash leaves both or neither. */
     async initChildWorkflowStatus(initStatus, creatorXid, parentWorkflowID, parentFunctionID, startTimeEpochMs, endTimeEpochMs, reusePolicy = 'return-existing') {
-        const client = await this.#connect();
+        const client = await this.connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
@@ -1070,7 +1021,7 @@ class SystemDatabase {
             'schedule_name',
             'application_name',
         ];
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Chunk to stay well under the bind-parameter limit.
@@ -1123,7 +1074,7 @@ class SystemDatabase {
         return inserted;
     }
     async recordWorkflowOutput(workflowID, status, ownerXid) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.SUCCESS, { output: status.output }, ownerXid);
         }
@@ -1132,7 +1083,7 @@ class SystemDatabase {
         }
     }
     async recordWorkflowError(workflowID, status, ownerXid) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             return await this.#recordWorkflowOutcome(client, workflowID, workflow_1.StatusString.ERROR, { error: status.error }, ownerXid);
         }
@@ -1180,22 +1131,11 @@ class SystemDatabase {
             }
         }
     }
-    async getPendingWorkflows(executorID, appVersion) {
-        const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
-        // executor_id defaults to "local", so it collides across applications.
-        const scope = this.#appNameFilter('application_name', this.appName, params);
-        const getWorkflows = await this.pool.query(`SELECT workflow_uuid
-       FROM "${this.schemaName}".workflow_status
-       WHERE status=$1 AND executor_id=$2 AND application_version=$3 AND ${scope}`, params);
-        return getWorkflows.rows.map((i) => ({
-            workflowUUID: i.workflow_uuid,
-        }));
-    }
     // Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
     async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
         const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
         // executor_id defaults to "local", so it collides across applications.
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET started_at_epoch_ms = NULL,
            status = $1,
@@ -1216,7 +1156,7 @@ class SystemDatabase {
             return status ? JSON.stringify(status) : null;
         };
         if (callerID && callerFN) {
-            const client = await this.#connect();
+            const client = await this.connect();
             try {
                 // Check if the operation has been done before for OAOO (only do this inside a workflow).
                 const json = await this.#inTransaction(client, () => this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus));
@@ -1252,7 +1192,7 @@ class SystemDatabase {
     }
     // Only used in tests
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await this.updateWorkflowStatus(client, workflowID, status, {
                 update: {
@@ -1270,7 +1210,7 @@ class SystemDatabase {
     }
     // ==================== Step Results ====================
     async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             return await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
         }
@@ -1278,8 +1218,13 @@ class SystemDatabase {
             client.release();
         }
     }
-    async getAllOperationResults(workflowID, limit, offset) {
-        let query = `SELECT * FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
+    async getAllOperationResults(workflowID, limit, offset, loadOutput = true) {
+        // Skip the output/error payload columns when the caller does not need them,
+        // mirroring how listWorkflows skips its payload joins when loadOutput is false.
+        const columns = loadOutput
+            ? '*'
+            : 'workflow_uuid, function_id, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp';
+        let query = `SELECT ${columns} FROM "${this.schemaName}".operation_outputs WHERE workflow_uuid=$1 ORDER BY function_id`;
         const params = [workflowID];
         if (limit !== undefined) {
             params.push(limit);
@@ -1293,7 +1238,7 @@ class SystemDatabase {
         return rows;
     }
     async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await this.#inTransaction(client, () => this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options));
         }
@@ -1302,31 +1247,71 @@ class SystemDatabase {
             await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
         }
     }
-    async runTransactionalStep(workflowID, functionID, functionName, callback) {
-        const client = await this.#connect();
+    /** Checkpoint an operation's failure so a replay re-throws it instead of re-running. */
+    async recordOperationError(workflowID, functionID, functionName, startTimeEpochMs, error, childWorkflowID) {
+        let serialized;
         try {
-            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
-            if (existing !== undefined) {
+            serialized = await (0, serialization_1.serializeResErrorWithSerializer)(error, this.serializer, this.serializer.name());
+            await (0, serialization_1.deserializeValue)(serialized.serializedValue, serialized.serialization, this.serializer);
+        }
+        catch (serError) {
+            // A checkpoint replay could not load would wedge the workflow; leave it unrecorded so replay re-runs.
+            this.logger.warn(`Not checkpointing ${functionName} failure in workflow ${workflowID}: its error cannot be serialized: ${String(serError)}`);
+            return;
+        }
+        await this.recordOperationResult(workflowID, functionID, functionName, true, startTimeEpochMs, Date.now(), {
+            error: serialized.serializedValue,
+            serialization: serialized.serialization,
+            childWorkflowID,
+        });
+    }
+    async runTransactionalStep(workflowID, functionID, functionName, callback) {
+        const startTime = Date.now();
+        // The latest attempt's callback failure, told apart from a check or checkpoint failure.
+        let callbackError = undefined;
+        const attempt = async () => {
+            callbackError = undefined;
+            const client = await this.connect();
+            try {
+                await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+                const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
+                if (existing !== undefined) {
+                    await client.query('ROLLBACK');
+                    return existing;
+                }
+                let output;
+                try {
+                    output = await callback(client);
+                }
+                catch (e) {
+                    callbackError = e;
+                    throw e;
+                }
+                await this.recordOperationResultInternal(client, workflowID, functionID, functionName, true, startTime, Date.now(), {
+                    output,
+                });
+                await client.query('COMMIT');
+                await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
+                return undefined;
+            }
+            catch (e) {
                 await client.query('ROLLBACK');
-                return existing;
+                throw e;
             }
-            const startTime = Date.now();
-            const output = await callback(client);
-            await this.recordOperationResultInternal(client, workflowID, functionID, functionName, true, startTime, Date.now(), {
-                output,
-            });
-            await client.query('COMMIT');
-            await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
-            return undefined;
+            finally {
+                client.release();
+            }
+        };
+        try {
+            return await withDbRetry(attempt);
         }
         catch (e) {
-            await client.query('ROLLBACK');
+            // Retries are done and the transaction rolled back, so record the callback's own failure separately.
+            if (e === callbackError) {
+                await this.recordOperationError(workflowID, functionID, functionName, startTime, e);
+            }
             throw e;
         }
-        finally {
-            client.release();
-        }
     }
     async checkPatch(workflowID, functionID, patchName, deprecated) {
         // Not doing a cancel check at this point.
@@ -1349,7 +1334,7 @@ class SystemDatabase {
         }
         // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
         //  If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             return await this.#inTransaction(client, async () => {
                 const checkpointName = await readCheckpointName(client);
@@ -1396,7 +1381,7 @@ class SystemDatabase {
         await this.#checkIfCanceled(this.pool, workflowID);
     }
     async resumeWorkflows(workflowIDs, queueName) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Check existence separately: a zero-row update also means "already complete", a legal no-op.
@@ -1460,7 +1445,7 @@ class SystemDatabase {
         return await this.debounceDelayedWorkflowStandalone(params);
     }
     async debounceDelayedWorkflowStandalone(params) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const result = await this.#debounceDelayedWorkflowInternal(client, params);
@@ -1488,7 +1473,7 @@ class SystemDatabase {
             params.applicationName ?? null,
         ];
         // Never extend a workflow the target application doesn't own; falls through to the holder below.
-        const ownScope = this.#appNameFilter('application_name', params.applicationName, updateParams);
+        const ownScope = this.appNameFilter('application_name', params.applicationName, updateParams);
         const updated = await client.query(`UPDATE "${this.schemaName}".workflow_status
        SET delay_until_epoch_ms = CASE
              WHEN debounce_deadline_epoch_ms IS NOT NULL AND debounce_deadline_epoch_ms < $1
@@ -1570,7 +1555,7 @@ class SystemDatabase {
                 allIds.push(...(await this.getWorkflowChildren(wfid)));
             }
         }
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // The payload tables carry no foreign key, so the status delete does not cascade
@@ -1616,7 +1601,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`startStep must be >= 0, got ${startStep}`);
         }
         const schema = this.schemaName;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             const { rows: statusRows } = await client.query(`SELECT status FROM "${schema}".workflow_status WHERE workflow_uuid = $1`, [workflowID]);
@@ -1698,62 +1683,6 @@ class SystemDatabase {
         const result = await this.bulkForkWorkflows([workflowID], [newWorkflowID], [startStep], options);
         return result[0];
     }
-    async forkFromFailure(workflowIDs, options = {}) {
-        const modes = [
-            options.fromLastFailure ?? false,
-            options.fromLastStep ?? false,
-            options.fromStep !== undefined,
-            options.fromStepName !== undefined,
-        ].filter(Boolean).length;
-        if (modes !== 1) {
-            throw new Error('Exactly one of fromLastFailure, fromLastStep, fromStep, or fromStepName must be specified');
-        }
-        let startSteps;
-        if (options.fromStep !== undefined) {
-            startSteps = Array(workflowIDs.length).fill(options.fromStep);
-        }
-        else {
-            let query;
-            const params = [workflowIDs];
-            if (options.fromLastFailure) {
-                query = `SELECT workflow_uuid,
-                        COALESCE(
-                          MAX(function_id) FILTER (WHERE error IS NOT NULL),
-                          MAX(function_id)
-                        ) AS start_step
-                 FROM "${this.schemaName}".operation_outputs
-                 WHERE workflow_uuid = ANY($1)
-                 GROUP BY workflow_uuid`;
-            }
-            else if (options.fromLastStep) {
-                query = `SELECT workflow_uuid, MAX(function_id) AS start_step
-                 FROM "${this.schemaName}".operation_outputs
-                 WHERE workflow_uuid = ANY($1)
-                 GROUP BY workflow_uuid`;
-            }
-            else {
-                // fromStepName
-                query = `SELECT workflow_uuid, MAX(function_id) AS start_step
-                 FROM "${this.schemaName}".operation_outputs
-                 WHERE workflow_uuid = ANY($1) AND function_name = $2
-                 GROUP BY workflow_uuid`;
-                params.push(options.fromStepName);
-            }
-            const result = await this.pool.query(query, params);
-            const startStepByID = new Map(result.rows.map((r) => [r.workflow_uuid, Number(r.start_step)]));
-            if (options.fromStepName !== undefined) {
-                for (const wid of workflowIDs) {
-                    if (!startStepByID.has(wid)) {
-                        throw new Error(`Workflow ${wid} has no step named '${options.fromStepName}'`);
-                    }
-                }
-            }
-            // A workflow with no recorded steps has nothing to resume from, so restart it from the beginning.
-            startSteps = workflowIDs.map((wid) => startStepByID.get(wid) ?? 0);
-        }
-        const forkedIDs = workflowIDs.map(() => (0, crypto_1.randomUUID)());
-        return this.bulkForkWorkflows(workflowIDs, forkedIDs, startSteps, options);
-    }
     async bulkForkWorkflows(originalWorkflowIDs, forkedWorkflowIDs, startSteps, options = {}) {
         if (originalWorkflowIDs.length === 0) {
             return [];
@@ -1761,7 +1690,7 @@ class SystemDatabase {
         if (originalWorkflowIDs.length !== forkedWorkflowIDs.length || originalWorkflowIDs.length !== startSteps.length) {
             throw new Error('originalWorkflowIDs, forkedWorkflowIDs, and startSteps must have the same length');
         }
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Fetch the status of all original workflows inside the transaction.
@@ -1913,188 +1842,6 @@ class SystemDatabase {
             client.release();
         }
     }
-    async exportWorkflow(workflowID, exportChildren = false) {
-        const workflowIDs = [workflowID];
-        if (exportChildren) {
-            workflowIDs.push(...(await this.getWorkflowChildren(workflowID)));
-        }
-        const exportedWorkflows = [];
-        const client = await this.#connect();
-        try {
-            for (const wfID of workflowIDs) {
-                // Export workflow_status
-                const statusResult = await client.query(
-                // creator_xid and owner_xid are intentionally omitted: they are transient
-                // tokens, not logical workflow state, and a source database's
-                // tokens are meaningless in the target.
-                `SELECT
-            ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
-            ws.authenticated_roles,
-            COALESCE(wo.output, ws.output) AS output, COALESCE(wo.error, ws.error) AS error,
-            ws.executor_id,
-            ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
-            ws.class_name, ws.config_name, ws.recovery_attempts, ws.queue_name,
-            ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms, ws.started_at_epoch_ms,
-            ws.deduplication_id, COALESCE(wi.inputs, ws.inputs) AS inputs,
-            ws.priority, ws.queue_partition_key, ws.forked_from,
-            ws.parent_workflow_id, ws.serialization, ws.delay_until_epoch_ms,
-            ws.was_forked_from, ws.rate_limited, ws.completed_at, ws.attributes, ws.schedule_name,
-            ws.debounce_deadline_epoch_ms, ws.is_debounced, ws.application_name
-          FROM "${this.schemaName}".workflow_status ws
-          LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
-          LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
-          WHERE ws.workflow_uuid = $1`, [wfID]);
-                if (statusResult.rows.length === 0) {
-                    throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${wfID} does not exist`);
-                }
-                const workflowStatus = statusResult.rows[0];
-                // Export operation_outputs
-                const outputsResult = await client.query(`SELECT
-            workflow_uuid, function_id, function_name, output, error,
-            child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
-            serialization, application_name
-          FROM "${this.schemaName}".operation_outputs
-          WHERE workflow_uuid = $1`, [wfID]);
-                // Export workflow_events
-                const eventsResult = await client.query(`SELECT workflow_uuid, key, value, serialization
-          FROM "${this.schemaName}".workflow_events
-          WHERE workflow_uuid = $1`, [wfID]);
-                // Export workflow_events_history
-                const historyResult = await client.query(`SELECT workflow_uuid, function_id, key, value, serialization
-          FROM "${this.schemaName}".workflow_events_history
-          WHERE workflow_uuid = $1`, [wfID]);
-                // Export streams
-                const streamsResult = await client.query(`SELECT workflow_uuid, key, value, "offset", function_id, serialization
-          FROM "${this.schemaName}".streams
-          WHERE workflow_uuid = $1`, [wfID]);
-                exportedWorkflows.push({
-                    workflow_status: workflowStatus,
-                    operation_outputs: outputsResult.rows,
-                    workflow_events: eventsResult.rows,
-                    workflow_events_history: historyResult.rows,
-                    streams: streamsResult.rows,
-                });
-            }
-        }
-        finally {
-            client.release();
-        }
-        return exportedWorkflows;
-    }
-    async importWorkflow(workflows) {
-        const client = await this.#connect();
-        try {
-            await client.query('BEGIN');
-            for (const workflow of workflows) {
-                const status = workflow.workflow_status;
-                // Import workflow_status
-                await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
-            workflow_uuid, status, name, authenticated_user, assumed_role,
-            authenticated_roles, output, error, executor_id,
-            created_at, updated_at, application_version, application_id,
-            class_name, config_name, recovery_attempts, queue_name,
-            workflow_timeout_ms, workflow_deadline_epoch_ms, started_at_epoch_ms,
-            deduplication_id, inputs, priority, queue_partition_key, forked_from,
-            parent_workflow_id, serialization, delay_until_epoch_ms,
-            was_forked_from, rate_limited, completed_at, attributes, schedule_name,
-            debounce_deadline_epoch_ms, is_debounced, application_name
-          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)`, [
-                    status.workflow_uuid,
-                    status.status,
-                    status.name,
-                    status.authenticated_user,
-                    status.assumed_role,
-                    status.authenticated_roles,
-                    // Legacy columns: the payloads live in their own tables.
-                    null,
-                    null,
-                    status.executor_id,
-                    status.created_at,
-                    status.updated_at,
-                    status.application_version,
-                    status.application_id,
-                    status.class_name,
-                    status.config_name,
-                    status.recovery_attempts,
-                    status.queue_name,
-                    status.workflow_timeout_ms,
-                    status.workflow_deadline_epoch_ms,
-                    status.started_at_epoch_ms,
-                    status.deduplication_id,
-                    null,
-                    status.priority,
-                    status.queue_partition_key,
-                    status.forked_from,
-                    status.parent_workflow_id,
-                    status.serialization,
-                    status.delay_until_epoch_ms ?? null,
-                    // NOT NULL columns: fall back to FALSE for payloads exported before
-                    // these fields were included.
-                    status.was_forked_from ?? false,
-                    status.rate_limited ?? false,
-                    status.completed_at ?? null,
-                    status.attributes ? JSON.stringify(status.attributes) : null,
-                    status.schedule_name ?? null,
-                    status.debounce_deadline_epoch_ms ?? null,
-                    status.is_debounced ?? false,
-                    status.application_name ?? null,
-                ]);
-                // Retention starts at import: the original timestamps are long past the cutoff
-                // and would be collected immediately.
-                await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
-           VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.inputs]);
-                if (status.output !== null || status.error !== null) {
-                    await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error, retention_timestamp)
-             VALUES ($1, $2, $3, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.output, status.error]);
-                }
-                // Import operation_outputs
-                for (const output of workflow.operation_outputs) {
-                    await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
-              workflow_uuid, function_id, function_name, output, error,
-              child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
-              serialization, application_name, retention_timestamp
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [
-                        output.workflow_uuid,
-                        output.function_id,
-                        output.function_name,
-                        output.output,
-                        output.error,
-                        output.child_workflow_id,
-                        output.started_at_epoch_ms,
-                        output.completed_at_epoch_ms,
-                        output.serialization,
-                        output.application_name ?? null,
-                    ]);
-                }
-                // Import workflow_events
-                for (const event of workflow.workflow_events) {
-                    await client.query(`INSERT INTO "${this.schemaName}".workflow_events (
-              workflow_uuid, key, value, serialization
-            ) VALUES ($1, $2, $3, $4)`, [event.workflow_uuid, event.key, event.value, event.serialization]);
-                }
-                // Import workflow_events_history
-                for (const history of workflow.workflow_events_history) {
-                    await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (
-              workflow_uuid, function_id, key, value, serialization
-            ) VALUES ($1, $2, $3, $4, $5)`, [history.workflow_uuid, history.function_id, history.key, history.value, history.serialization]);
-                }
-                // Import streams
-                for (const stream of workflow.streams) {
-                    await client.query(`INSERT INTO "${this.schemaName}".streams (
-              workflow_uuid, key, value, "offset", function_id, serialization
-            ) VALUES ($1, $2, $3, $4, $5, $6)`, [stream.workflow_uuid, stream.key, stream.value, stream.offset, stream.function_id, stream.serialization]);
-                }
-            }
-            await client.query('COMMIT');
-        }
-        catch (error) {
-            await client.query('ROLLBACK');
-            throw error;
-        }
-        finally {
-            client.release();
-        }
-    }
     // ==================== Awaiting Workflows ====================
     /** Register one execution; the returned release is idempotent, so it can run both before a park and once the run settles. */
     registerRunningWorkflow(workflowID, queueName, queuePartitionKey) {
@@ -2382,7 +2129,9 @@ class SystemDatabase {
     async send(workflowID, functionID, destinationID, message, topic, serialization, idempotencyKey) {
         topic = topic ?? this.nullTopic;
         const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
-        const client = await this.#connect();
+        const startTime = Date.now();
+        let missing;
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
@@ -2398,7 +2147,7 @@ class SystemDatabase {
             const err = error;
             if (err.code === '23503') {
                 // Foreign key constraint violation (only expected for the INSERT query)
-                throw new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
+                missing = new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
             }
             else {
                 throw err;
@@ -2407,6 +2156,11 @@ class SystemDatabase {
         finally {
             client.release();
         }
+        if (missing) {
+            // A missing destination is the send's answer; the transaction has rolled back, so record it separately.
+            await this.recordOperationError(workflowID, functionID, exports.DBOS_FUNCNAME_SEND, startTime, missing);
+            throw missing;
+        }
     }
     /** Runs on `client` if given, joining its transaction; otherwise on the pool with retries. */
     async sendDirect(destinationID, message, topic, serialization, idempotencyKey, client) {
@@ -2418,7 +2172,7 @@ class SystemDatabase {
     /** A send from inside a step of `workflowID`: not recorded, but it lands only while the caller still owns the workflow. */
     async sendFromStep(workflowID, destinationID, message, topic, serialization, idempotencyKey) {
         const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await this.#inTransaction(client, async () => {
                 await this.#sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey);
@@ -2506,7 +2260,7 @@ class SystemDatabase {
         // Transactionally consume and return the message if it's in the DB, otherwise return null.
         let message = null;
         let serialization = null;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
             const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
@@ -2546,7 +2300,7 @@ class SystemDatabase {
     }
     // ==================== Events ====================
     async setEvent(workflowID, functionID, key, message, serialization) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Only a real write (not a replay) should wake readers.
@@ -2654,7 +2408,7 @@ class SystemDatabase {
     // ==================== Streams ====================
     async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
         const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             while (true) {
                 try {
@@ -2691,7 +2445,7 @@ class SystemDatabase {
         }
     }
     async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             while (true) {
                 // Only a real insert (not a replay) should wake readers.
@@ -2825,7 +2579,7 @@ class SystemDatabase {
             }
             try {
                 // One statement: one round trip, one async-notify queue-lock acquisition; unnest emits one notification per payload.
-                const client = await this.#connect();
+                const client = await this.connect();
                 try {
                     await client.query(`SELECT pg_notify($1, p) FROM unnest($2::text[]) AS p`, [channel, Array.from(batch)]);
                 }
@@ -2840,49 +2594,6 @@ class SystemDatabase {
         }
     }
     // ==================== Observability: Workflow Communications ====================
-    async getAllEvents(workflowID) {
-        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
-         WHERE workflow_uuid = $1`, [workflowID]));
-        const events = {};
-        for (const row of rows) {
-            events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
-        }
-        return events;
-    }
-    async getAllNotifications(workflowID) {
-        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
-         FROM "${this.schemaName}".notifications
-         WHERE destination_uuid = $1
-         ORDER BY created_at_epoch_ms`, [workflowID]));
-        return await Promise.all(rows.map(async (row) => ({
-            topic: row.topic === this.nullTopic ? null : row.topic,
-            message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
-            createdAtEpochMs: Number(row.created_at_epoch_ms),
-            consumed: row.consumed,
-        })));
-    }
-    async getAllStreamEntries(workflowID) {
-        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
-         WHERE workflow_uuid = $1
-         ORDER BY key, "offset"`, [workflowID]));
-        const streams = {};
-        const closed = new Set();
-        for (const row of rows) {
-            if (closed.has(row.key)) {
-                continue;
-            }
-            // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
-            const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
-            if (isStreamClosedSentinel(value)) {
-                // End the stream where readStream does, so the two never disagree.
-                closed.add(row.key);
-                streams[row.key] ??= [];
-                continue;
-            }
-            (streams[row.key] ??= []).push(value);
-        }
-        return streams;
-    }
     // ==================== Queues ====================
     async transitionDelayedWorkflows() {
         // Transition workflows from DELAYED to ENQUEUED when their delay has expired.
@@ -2890,7 +2601,7 @@ class SystemDatabase {
         // debounce key held only while DELAYED, so a later same-key debounce starts a fresh workflow.
         const params = [workflow_1.StatusString.ENQUEUED, Date.now(), workflow_1.StatusString.DELAYED];
         // Only what this application would dequeue: a peer's debounce key is not ours to clear.
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
@@ -2899,7 +2610,7 @@ class SystemDatabase {
     /** Cancel up to `limit` of this application's active workflows whose deadline has passed, returning their IDs. */
     async cancelTimedOutWorkflows(limit) {
         const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
         // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
         const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
@@ -2930,7 +2641,7 @@ class SystemDatabase {
         // Recursive-CTE loose index scan: SELECT DISTINCT would scan every ENQUEUED row, whereas each iteration here is one seek on idx_workflow_status_partition_dequeue_v3, so cost scales with the number of partitions rather than the backlog depth.
         const params = [queueName, workflow_1.StatusString.ENQUEUED];
         // Only partitions this application can actually dequeue from.
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         const { rows } = await this.pool.query(`WITH RECURSIVE partitions AS (
          (SELECT MIN(queue_partition_key) AS pk
           FROM "${this.schemaName}".workflow_status
@@ -2957,7 +2668,7 @@ class SystemDatabase {
             queue.partitionRateLimit !== undefined;
         // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
         const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             // Default to READ COMMITTED except with a budget shared across executors
             if (hasSharedBudget) {
@@ -2970,7 +2681,7 @@ class SystemDatabase {
             const rateLimitRemaining = async (rateLimit, partitionScoped) => {
                 const params = [queue.name, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED, rateLimit.periodSec * 1000];
                 // Count only what this application would dequeue, matching the select below.
-                const scope = this.#appNameFilter('application_name', this.appName, params);
+                const scope = this.appNameFilter('application_name', this.appName, params);
                 const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
                 const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
            WHERE queue_name = $1
@@ -2989,7 +2700,7 @@ class SystemDatabase {
              */
             const pendingCount = async (partitionScoped) => {
                 const params = [queue.name, workflow_1.StatusString.PENDING];
-                const scope = this.#appNameFilter('application_name', this.appName, params);
+                const scope = this.appNameFilter('application_name', this.appName, params);
                 const partitionFilter = partitionScoped ? `AND queue_partition_key = $${params.push(queuePartitionKey)}` : '';
                 const { rows } = await client.query(`SELECT COUNT(*) FROM "${this.schemaName}".workflow_status
            WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
@@ -3050,7 +2761,7 @@ class SystemDatabase {
             const lockMode = hasSharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
             const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
             const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
-            const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
+            const selectScope = this.appNameFilter('application_name', this.appName, selectParams);
             const selectQuery = `
         SELECT workflow_uuid
         FROM "${this.schemaName}".workflow_status
@@ -3083,7 +2794,7 @@ class SystemDatabase {
                     ownerXid,
                 ];
                 // Re-check ownership alongside status, as the partitioned claim guard does.
-                const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
+                const claimScope = this.appNameFilter('application_name', this.appName, updateParams);
                 // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
                 const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
            SET status = $1,
@@ -3133,7 +2844,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with partition concurrency 1 and no queue-wide concurrency or rate limit: ${queue.name}`);
         }
         // partitionWorkerConcurrency needs no handling here: it cannot exceed partition concurrency 1, which the PENDING gate already enforces globally.
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN');
             const latestVersion = await this.#latestApplicationVersionName(client);
@@ -3152,7 +2863,7 @@ class SystemDatabase {
                 workflow_1.StatusString.PENDING,
                 sweepLimit,
             ];
-            const candidateScope = this.#appNameFilter('application_name', this.appName, candidateParams);
+            const candidateScope = this.appNameFilter('application_name', this.appName, candidateParams);
             // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
             const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
            (SELECT MIN(queue_partition_key) AS pk
@@ -3209,7 +2920,7 @@ class SystemDatabase {
            AND ${versionClause(version)}
            AND ${scope}`;
             const lockedParams = [candidateIDs, workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
-            const lockedScope = this.#appNameFilter('application_name', this.appName, lockedParams);
+            const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
             // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
             const lockedResult = await client.query(`SELECT workflow_uuid
          FROM "${this.schemaName}".workflow_status
@@ -3233,7 +2944,7 @@ class SystemDatabase {
                 this.appName ?? null,
                 ownerXid,
             ];
-            const flipScope = this.#appNameFilter('application_name', this.appName, flipParams);
+            const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
             // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
             const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
          SET status = $1,
@@ -3359,8 +3070,8 @@ class SystemDatabase {
         // Falsy, not just undefined: a Conductor request carries an omitted list as JSON null.
         const idKeyed = (input.workflowIDs?.length ?? 0) > 0;
         whereClauses.push(idKeyed
-            ? this.#appNameFilter('application_name', input.applicationName, params)
-            : this.#observabilityFilter('application_name', input.applicationName, params));
+            ? this.appNameFilter('application_name', input.applicationName, params)
+            : this.observabilityFilter('application_name', input.applicationName, params));
         paramCounter = params.length + 1;
         if (input.workflow_id_prefix) {
             if (Array.isArray(input.workflow_id_prefix)) {
@@ -3389,6 +3100,10 @@ class SystemDatabase {
             params.push(input.wasForkedFrom);
             paramCounter++;
         }
+        // Matches the forks themselves, as opposed to wasForkedFrom, which matches the workflows they were forked from.
+        if (input.isFork !== undefined) {
+            whereClauses.push(input.isFork ? `forked_from IS NOT NULL` : `forked_from IS NULL`);
+        }
         if (input.hasParent !== undefined) {
             if (input.hasParent) {
                 whereClauses.push(`parent_workflow_id IS NOT NULL`);
@@ -3469,630 +3184,6 @@ class SystemDatabase {
             : await this.observabilityQuery((client) => client.query(query, params));
         return result.rows.map(mapWorkflowStatus);
     }
-    async getWorkflowAggregates(input) {
-        if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
-            throw new Error('time_bucket_size_ms must be > 0');
-        }
-        const groupByFlags = [
-            ['status', input.groupByStatus ?? false, 'status'],
-            ['name', input.groupByName ?? false, 'name'],
-            ['queue_name', input.groupByQueueName ?? false, 'queue_name'],
-            ['executor_id', input.groupByExecutorId ?? false, 'executor_id'],
-            ['application_version', input.groupByApplicationVersion ?? false, 'application_version'],
-            ['application_name', input.groupByApplicationName ?? false, 'application_name'],
-        ];
-        const groupNames = [];
-        const groupColumns = [];
-        const groupSelectColumns = [];
-        for (const [colName, enabled, col] of groupByFlags) {
-            if (enabled) {
-                groupNames.push(colName);
-                groupColumns.push(col);
-                groupSelectColumns.push(col);
-            }
-        }
-        const params = [];
-        if (input.timeBucketSizeMs !== undefined) {
-            // Bucket on created_at — the indexed wall-clock timestamp on workflow_status.
-            // One placeholder shared by SELECT and GROUP BY, so Postgres sees the two expressions as identical.
-            params.push(input.timeBucketSizeMs);
-            const bucket = `$${params.length}::bigint`;
-            const bucketExpr = `(CAST(FLOOR(created_at / ${bucket}) AS BIGINT) * ${bucket})`;
-            groupNames.push('time_bucket');
-            groupColumns.push(bucketExpr);
-            groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
-        }
-        if (groupColumns.length === 0) {
-            throw new Error('At least one group_by flag must be set to True');
-        }
-        // Build select columns from boolean flags. MAX ignores NULLs, so rows
-        // missing started_at_epoch_ms or completed_at naturally drop out of the
-        // latency maxes.
-        const selectFlags = [
-            ['count', input.selectCount ?? false, 'COUNT(*)'],
-            ['min_created_at', input.selectMinCreatedAt ?? false, 'MIN(created_at)'],
-            ['max_queue_wait_ms', input.selectMaxQueueWaitMs ?? false, 'MAX(started_at_epoch_ms - created_at)'],
-            ['max_total_latency_ms', input.selectMaxTotalLatencyMs ?? false, 'MAX(completed_at - created_at)'],
-        ];
-        const selectNames = [];
-        const selectColumns = [];
-        for (const [name, enabled, expr] of selectFlags) {
-            if (enabled) {
-                selectNames.push(name);
-                selectColumns.push(`${expr} AS ${name}`);
-            }
-        }
-        if (selectColumns.length === 0) {
-            throw new Error('At least one select_ flag must be set to True');
-        }
-        const whereClauses = [];
-        let paramIdx = params.length + 1;
-        const addFilter = (column, values) => {
-            if (!values || values.length === 0)
-                return;
-            const placeholders = values.map((_, i) => `$${paramIdx + i}`).join(', ');
-            whereClauses.push(`${column} IN (${placeholders})`);
-            params.push(...values);
-            paramIdx += values.length;
-        };
-        addFilter('status', input.status);
-        addFilter('name', input.name);
-        addFilter('application_version', input.appVersion);
-        addFilter('executor_id', input.executorId);
-        addFilter('queue_name', input.queueName);
-        if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
-            const likeClauses = input.workflowIdPrefix.map((p) => {
-                params.push(`${p}%`);
-                return `workflow_uuid LIKE $${paramIdx++}`;
-            });
-            whereClauses.push(`(${likeClauses.join(' OR ')})`);
-        }
-        addFilter('workflow_uuid', input.workflowIDs);
-        addFilter('authenticated_user', input.authenticatedUser);
-        addFilter('forked_from', input.forkedFrom);
-        addFilter('parent_workflow_id', input.parentWorkflowID);
-        addFilter('schedule_name', input.scheduleName);
-        // Unset scopes to this application, as on every other observability query.
-        whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
-        paramIdx = params.length + 1;
-        if (input.wasForkedFrom !== undefined) {
-            whereClauses.push(`was_forked_from = $${paramIdx}`);
-            params.push(input.wasForkedFrom);
-            paramIdx++;
-        }
-        if (input.hasParent !== undefined) {
-            whereClauses.push(input.hasParent ? `parent_workflow_id IS NOT NULL` : `parent_workflow_id IS NULL`);
-        }
-        // Match workflows whose attributes JSONB contains all the given key-value pairs.
-        if (input.attributes && Object.keys(input.attributes).length > 0) {
-            whereClauses.push(`attributes @> $${paramIdx}::jsonb`);
-            params.push(JSON.stringify(input.attributes));
-            paramIdx++;
-        }
-        if (input.startTime) {
-            whereClauses.push(`created_at >= $${paramIdx}`);
-            params.push(new Date(input.startTime).getTime());
-            paramIdx++;
-        }
-        if (input.endTime) {
-            whereClauses.push(`created_at <= $${paramIdx}`);
-            params.push(new Date(input.endTime).getTime());
-            paramIdx++;
-        }
-        if (input.completedAfter) {
-            whereClauses.push(`completed_at >= $${paramIdx}`);
-            params.push(new Date(input.completedAfter).getTime());
-            paramIdx++;
-        }
-        if (input.completedBefore) {
-            whereClauses.push(`completed_at <= $${paramIdx}`);
-            params.push(new Date(input.completedBefore).getTime());
-            paramIdx++;
-        }
-        // dequeuedAfter/Before filter on started_at_epoch_ms: that column is
-        // populated on dequeue and surfaced as WorkflowStatus.dequeuedAt.
-        if (input.dequeuedAfter) {
-            whereClauses.push(`started_at_epoch_ms >= $${paramIdx}`);
-            params.push(new Date(input.dequeuedAfter).getTime());
-            paramIdx++;
-        }
-        if (input.dequeuedBefore) {
-            whereClauses.push(`started_at_epoch_ms <= $${paramIdx}`);
-            params.push(new Date(input.dequeuedBefore).getTime());
-            paramIdx++;
-        }
-        const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
-        const groupByClause = groupColumns.join(', ');
-        const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
-        const query = `
-      SELECT ${selectClause}
-      FROM "${this.schemaName}".workflow_status
-      ${whereClause}
-      GROUP BY ${groupByClause}
-    `;
-        const result = await this.observabilityQuery((client) => client.query(query, params));
-        const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
-        return result.rows.map((row) => {
-            const group = {};
-            for (const name of groupNames) {
-                const v = row[name];
-                group[name] = v === null || v === undefined ? null : String(v);
-            }
-            return {
-                group,
-                count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
-                minCreatedAt: selectNames.includes('min_created_at') ? toIntOrNull(row.min_created_at) : null,
-                maxQueueWaitMs: selectNames.includes('max_queue_wait_ms') ? toIntOrNull(row.max_queue_wait_ms) : null,
-                maxTotalLatencyMs: selectNames.includes('max_total_latency_ms') ? toIntOrNull(row.max_total_latency_ms) : null,
-            };
-        });
-    }
-    async getStepAggregates(input) {
-        if (input.timeBucketSizeMs !== undefined && input.timeBucketSizeMs <= 0) {
-            throw new Error('time_bucket_size_ms must be > 0');
-        }
-        // operation_outputs has no explicit status column; derive it from whether `error` is populated.
-        // Child-workflow mapping rows have NULL error, so they appear as SUCCESS — callers filter by function_name.
-        const statusExpr = `CASE WHEN error IS NULL THEN 'SUCCESS' ELSE 'ERROR' END`;
-        const groupByFlags = [
-            ['function_name', input.groupByFunctionName ?? false, 'function_name'],
-            ['status', input.groupByStatus ?? false, statusExpr],
-        ];
-        const groupNames = [];
-        const groupColumns = [];
-        const groupSelectColumns = [];
-        for (const [colName, enabled, expr] of groupByFlags) {
-            if (enabled) {
-                groupNames.push(colName);
-                groupColumns.push(expr);
-                groupSelectColumns.push(`${expr} AS ${colName}`);
-            }
-        }
-        const params = [];
-        if (input.timeBucketSizeMs !== undefined) {
-            // Bucket on completed_at_epoch_ms — it's the indexed timestamp on
-            // this table.
-            params.push(input.timeBucketSizeMs);
-            const bucket = `$${params.length}::bigint`;
-            const bucketExpr = `(CAST(FLOOR(completed_at_epoch_ms / ${bucket}) AS BIGINT) * ${bucket})`;
-            groupNames.push('time_bucket');
-            groupColumns.push(bucketExpr);
-            groupSelectColumns.push(`${bucketExpr} AS time_bucket`);
-        }
-        if (groupColumns.length === 0) {
-            throw new Error('At least one group_by flag must be set to True');
-        }
-        // Build select columns from boolean flags. Child-workflow mapping rows record start and
-        // complete at nearly the same instant, so they contribute ~0; DBOS.getResult and DBOS.sleep
-        // rows span their whole wait, so those dominate the duration max.
-        const selectFlags = [
-            ['count', input.selectCount ?? false, 'COUNT(*)'],
-            ['max_duration_ms', input.selectMaxDurationMs ?? false, 'MAX(completed_at_epoch_ms - started_at_epoch_ms)'],
-        ];
-        const selectNames = [];
-        const selectColumns = [];
-        for (const [name, enabled, expr] of selectFlags) {
-            if (enabled) {
-                selectNames.push(name);
-                selectColumns.push(`${expr} AS ${name}`);
-            }
-        }
-        if (selectColumns.length === 0) {
-            throw new Error('At least one select_ flag must be set to True');
-        }
-        const whereClauses = [];
-        let paramIdx = params.length + 1;
-        if (input.status && input.status.length > 0) {
-            const placeholders = input.status.map((_, i) => `$${paramIdx + i}`).join(', ');
-            whereClauses.push(`(${statusExpr}) IN (${placeholders})`);
-            params.push(...input.status);
-            paramIdx += input.status.length;
-        }
-        if (input.functionName && input.functionName.length > 0) {
-            const placeholders = input.functionName.map((_, i) => `$${paramIdx + i}`).join(', ');
-            whereClauses.push(`function_name IN (${placeholders})`);
-            params.push(...input.functionName);
-            paramIdx += input.functionName.length;
-        }
-        if (input.workflowIdPrefix && input.workflowIdPrefix.length > 0) {
-            const likeClauses = input.workflowIdPrefix.map((p) => {
-                params.push(`${p}%`);
-                return `workflow_uuid LIKE $${paramIdx++}`;
-            });
-            whereClauses.push(`(${likeClauses.join(' OR ')})`);
-        }
-        if (input.completedAfter) {
-            whereClauses.push(`completed_at_epoch_ms >= $${paramIdx}`);
-            params.push(new Date(input.completedAfter).getTime());
-            paramIdx++;
-        }
-        if (input.completedBefore) {
-            whereClauses.push(`completed_at_epoch_ms <= $${paramIdx}`);
-            params.push(new Date(input.completedBefore).getTime());
-            paramIdx++;
-        }
-        // Unset scopes to this application, as on every other observability query.
-        whereClauses.push(this.#observabilityFilter('application_name', input.applicationName, params));
-        paramIdx = params.length + 1;
-        const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
-        const groupByClause = groupColumns.join(', ');
-        const selectClause = [...groupSelectColumns, ...selectColumns].join(', ');
-        const query = `
-      SELECT ${selectClause}
-      FROM "${this.schemaName}".operation_outputs
-      ${whereClause}
-      GROUP BY ${groupByClause}
-    `;
-        const result = await this.observabilityQuery((client) => client.query(query, params));
-        const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
-        return result.rows.map((row) => {
-            const group = {};
-            for (const name of groupNames) {
-                const v = row[name];
-                group[name] = v === null || v === undefined ? null : String(v);
-            }
-            return {
-                group,
-                count: selectNames.includes('count') ? toIntOrNull(row.count) : null,
-                maxDurationMs: selectNames.includes('max_duration_ms') ? toIntOrNull(row.max_duration_ms) : null,
-            };
-        });
-    }
-    /** Whether the system database is CockroachDB. Resolved once, since detecting it costs a query. */
-    async #cockroach() {
-        if (this.#isCockroach === undefined) {
-            const client = await this.#connect();
-            try {
-                this.#isCockroach = await isCockroachDB(client);
-            }
-            finally {
-                client.release();
-            }
-        }
-        return this.#isCockroach;
-    }
-    /**
-     * Take a database-wide lock for one retention round, returning how to release it, or
-     * undefined when another round already holds it. The lock is session-scoped, so a round
-     * that crashes releases it. CockroachDB has no advisory locks and always takes it, so it
-     * collects unprotected rather than not at all.
-     */
-    async acquireRetentionLock() {
-        if (await this.#cockroach()) {
-            return { release: () => Promise.resolve() };
-        }
-        const key = retentionLockKey(this.schemaName).toString();
-        // The round holds this connection until it ends: releasing it would drop the lock.
-        const client = await this.#borrowRetentionClient();
-        let acquired = false;
-        try {
-            const { rows } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [key]);
-            acquired = rows[0]?.locked === true;
-        }
-        catch (e) {
-            this.#releaseRetentionClient(client);
-            throw e;
-        }
-        if (!acquired) {
-            this.#releaseRetentionClient(client);
-            return undefined;
-        }
-        return {
-            release: async () => {
-                // Already cut by destroy(): the session is gone and took the lock with it.
-                if (!this.#retentionClients.has(client)) {
-                    return;
-                }
-                try {
-                    // Explicit, since releasing the client only returns the session to the pool.
-                    const { rows } = await client.query('SELECT pg_advisory_unlock($1) AS released', [
-                        key,
-                    ]);
-                    if (rows[0]?.released !== true) {
-                        // False means this session no longer holds it, which a transaction-pooling proxy
-                        // causes by switching backends.
-                        this.logger.warn('Could not release the retention lock: this session no longer holds it. Retention will ' +
-                            'not proceed until the lock is released, which happens when the holding backend closes. ' +
-                            'A transaction-pooling proxy in front of Postgres causes this; run DBOS through a ' +
-                            'session-pooled or direct connection.');
-                    }
-                }
-                finally {
-                    this.#releaseRetentionClient(client);
-                }
-            },
-        };
-    }
-    /** VACUUM the tables a sweep dirtied. No-op where there is no autovacuum to outrun. */
-    async #vacuumTables(tables) {
-        if (await this.#cockroach()) {
-            return;
-        }
-        const client = await this.#borrowRetentionClient();
-        let notices = [];
-        const onNotice = (notice) => notices.push(notice.message ?? '');
-        client.on('notice', onNotice);
-        try {
-            for (const table of tables) {
-                // Per table, so one refusal does not skip the rest.
-                notices = [];
-                try {
-                    await client.query(`VACUUM (INDEX_CLEANUP ON, TRUNCATE OFF, ANALYZE) "${this.schemaName}"."${table}"`);
-                }
-                catch (e) {
-                    if (!this.#retentionClients.has(client)) {
-                        throw e;
-                    }
-                    this.logger.warn(`Payload retention could not vacuum ${table}: ${e.message}`);
-                    continue;
-                }
-                // A refused or stalled VACUUM does not raise, it says so in a notice; a successful
-                // one is silent, so anything here is worth surfacing.
-                for (const notice of notices) {
-                    this.logger.warn(`Payload retention vacuuming ${table}: ${notice}`);
-                }
-            }
-        }
-        finally {
-            client.removeListener('notice', onNotice);
-            this.#releaseRetentionClient(client);
-        }
-    }
-    /** Delete one payload table's orphans below the cutoff, one batch per transaction. */
-    async #garbageCollectTable(table, cutoff, batchSize) {
-        // A payload below the cutoff belongs to a workflow created before it, so the status side
-        // is the few such rows still present, not the whole table.
-        const orphaned = `NOT EXISTS (
-             SELECT 1 FROM "${this.schemaName}".workflow_status ws
-             WHERE ws.workflow_uuid = t.workflow_uuid AND ws.created_at < $1
-           )`;
-        // Seed from the oldest row in range.
-        const oldest = await retryOnSerializationError(async () => {
-            const { rows } = await this.pool.query(`SELECT retention_timestamp
-           FROM "${this.schemaName}".${table}
-          WHERE retention_timestamp < $1
-          ORDER BY retention_timestamp
-          LIMIT 1`, [cutoff]);
-            // retention_timestamp is a bigint, so node-postgres hands it back as a string.
-            return rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
-        });
-        if (oldest === undefined) {
-            return 0;
-        }
-        let total = 0;
-        let watermark = oldest - 1;
-        for (;;) {
-            const batch = await retryOnSerializationError(async () => {
-                // Borrowed rather than pool.query'd: that releases with the error, which discards the
-                // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
-                const client = await this.#borrowRetentionClient();
-                try {
-                    // Batches are cut by candidate count, so rows spared by the anti-join only thin one
-                    // out; they are re-checked next round.
-                    const { rows } = await client.query(`SELECT retention_timestamp
-               FROM "${this.schemaName}".${table}
-              WHERE retention_timestamp < $1 AND retention_timestamp > $2
-              ORDER BY retention_timestamp
-              LIMIT 1 OFFSET ${batchSize - 1}`, [cutoff, watermark]);
-                    const step = rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
-                    const params = [cutoff, watermark];
-                    // Timestamp ties may push the batch slightly over batchSize, but never split across two.
-                    const upperBound = step === undefined ? '' : `AND t.retention_timestamp <= $${params.push(step)} `;
-                    const result = await client.query(`DELETE FROM "${this.schemaName}".${table} t
-             WHERE t.retention_timestamp < $1 AND t.retention_timestamp > $2 ${upperBound}AND ${orphaned}`, params);
-                    return { step, deleted: result.rowCount ?? 0 };
-                }
-                finally {
-                    this.#releaseRetentionClient(client);
-                }
-            });
-            total += batch.deleted;
-            if (batch.step === undefined) {
-                return total;
-            }
-            watermark = batch.step;
-        }
-    }
-    /**
-     * Delete payload and step rows below the cutoff whose workflow is gone, returning the count
-     * removed from each table. Runs after the status sweep, whose orphans all fall in range:
-     * every payload is stamped no later than the completion that made the row collectable.
-     */
-    async garbageCollectPayloads(cutoff, batchSize = exports.DEFAULT_GC_BATCH_SIZE) {
-        // A NaN survives a bare `< 1` test and would only fail once it reached SQL.
-        if (!Number.isInteger(batchSize) || batchSize < 1) {
-            throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
-        }
-        const tables = ['workflow_input', 'workflow_output', 'operation_outputs'];
-        // To optimize performance, vacuum payload tables both before and after collecting them.
-        await this.#vacuumTables(['workflow_status', ...tables]);
-        // One connection per concurrent sweep, on top of the one the retention lock holds for the
-        // whole round. A pool too small for all three runs them sequentially rather than leaving
-        // sweeps waiting on a connection that only the round itself would free.
-        const poolMax = this.pool.options.max ?? exports.DEFAULT_POOL_SIZE;
-        const concurrency = Math.max(1, Math.min(tables.length, poolMax - 1));
-        const deleted = new Array(tables.length).fill(0);
-        const failures = [];
-        let next = 0;
-        const sweep = async () => {
-            for (;;) {
-                const i = next++;
-                if (i >= tables.length)
-                    return;
-                try {
-                    deleted[i] = await this.#garbageCollectTable(tables[i], cutoff, batchSize);
-                }
-                catch (e) {
-                    failures.push(e);
-                }
-            }
-        };
-        await Promise.all(Array.from({ length: concurrency }, () => sweep()));
-        // Only the first can be thrown, so the rest would otherwise be lost.
-        for (const extra of failures.slice(1)) {
-            this.logger.warn(`Payload retention sweep also failed: ${extra instanceof Error ? extra.message : String(extra)}`);
-        }
-        if (failures.length > 0) {
-            throw failures[0];
-        }
-        await this.#vacuumTables(tables);
-        this.logger.debug(`Payload retention deleted ${deleted[0]} inputs, ${deleted[1]} outputs, and ${deleted[2]} steps`);
-        return deleted;
-    }
-    /**
-     * Rows garbage collection may delete. completed_at is set on every terminal transition and
-     * cleared on resume, so one predicate covers eligibility: in-flight rows hold NULL and never
-     * compare true. Unscoped by application: retention is system-wide.
-     */
-    #gcFilter(cutoffEpochTimestampMs, params) {
-        params.push(cutoffEpochTimestampMs);
-        return `completed_at < $${params.length}`;
-    }
-    /**
-     * Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
-     * The delete is its own transaction; it re-checks the filter, so it needs no snapshot shared
-     * with the select that bounds it.
-     */
-    async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
-        // Borrowed rather than pool.query'd: that releases with the error, which discards the
-        // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
-        const client = await this.#borrowRetentionClient();
-        try {
-            // The batchSize-th oldest eligible row above the watermark bounds this range
-            const stepParams = [];
-            const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
-            stepParams.push(watermark);
-            const stepResult = await client.query(`SELECT completed_at
-           FROM "${this.schemaName}".workflow_status
-          WHERE ${stepScope} AND completed_at > $${stepParams.length}
-          ORDER BY completed_at
-          LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
-            // completed_at is a bigint, so node-postgres hands it back as a string.
-            const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].completed_at) : undefined;
-            const deleteParams = [];
-            let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
-            if (step !== undefined) {
-                // Inclusive upper bound: completed_at ties may push a batch over batchSize, but never split across two.
-                deleteParams.push(watermark, step);
-                deleteScope = `${deleteScope} AND completed_at > $${deleteParams.length - 1} AND completed_at <= $${deleteParams.length}`;
-            }
-            // The final batch drops the watermark: unbounded, since an import can land a
-            // completed_at below it mid-pass.
-            await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
-            return step;
-        }
-        finally {
-            this.#releaseRetentionClient(client);
-        }
-    }
-    /**
-     * Delete old terminal workflows throughout the system database, returning the cutoff
-     * actually used, or undefined when there is nothing to collect.
-     *
-     * Conductor sends cleared retention thresholds as JSON null, so every param is nullish.
-     */
-    async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
-        const batchSize = options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE;
-        // A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
-        if (!Number.isInteger(batchSize) || batchSize < 1) {
-            throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
-        }
-        if (rowsThreshold !== undefined && rowsThreshold !== null) {
-            // The completed_at of the rowsThreshold newest completed row
-            const result = await retryOnSerializationError(() => this.pool.query(`SELECT completed_at
-         FROM "${this.schemaName}".workflow_status
-         WHERE completed_at IS NOT NULL
-         ORDER BY completed_at DESC
-         LIMIT 1 OFFSET $1`, [rowsThreshold - 1]));
-            if (result.rows.length > 0) {
-                const rowsBasedCutoff = Number(result.rows[0].completed_at);
-                // Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
-                if (cutoffEpochTimestampMs === undefined ||
-                    cutoffEpochTimestampMs === null ||
-                    rowsBasedCutoff > cutoffEpochTimestampMs) {
-                    cutoffEpochTimestampMs = rowsBasedCutoff;
-                }
-            }
-        }
-        if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
-            return undefined;
-        }
-        // Narrowed to a constant so the closures below keep it.
-        const cutoff = cutoffEpochTimestampMs;
-        // Advance a completed_at watermark, one committed transaction per batch, so a long
-        // history neither deletes in one transaction nor rescans what it already deleted.
-        const oldest = await retryOnSerializationError(async () => {
-            const params = [];
-            const scope = this.#gcFilter(cutoff, params);
-            const { rows } = await this.pool.query(`SELECT completed_at
-           FROM "${this.schemaName}".workflow_status
-          WHERE ${scope}
-          ORDER BY completed_at
-          LIMIT 1`, params);
-            return rows.length > 0 ? Number(rows[0].completed_at) : undefined;
-        });
-        let watermark = oldest === undefined ? 0 : oldest - 1;
-        for (;;) {
-            const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
-            // Fewer than a full batch remained, so that delete took the rest.
-            if (next === undefined)
-                return cutoff;
-            watermark = next;
-        }
-    }
-    /**
-     * IDs of this application's in-flight workflows created at or before the cutoff.
-     * Claiming-scoped, so an upgrade still times out its own unclaimed workflows.
-     */
-    async listTimedOutWorkflowIds(cutoffEpochTimestampMs) {
-        const params = [
-            cutoffEpochTimestampMs,
-            workflow_1.StatusString.PENDING,
-            workflow_1.StatusString.ENQUEUED,
-            workflow_1.StatusString.DELAYED,
-        ];
-        const scope = this.#appNameFilter('application_name', this.appName, params);
-        const { rows } = await this.pool.query(`SELECT workflow_uuid
-       FROM "${this.schemaName}".workflow_status
-       WHERE created_at <= $1
-         AND status IN ($2, $3, $4)
-         AND ${scope}`, params);
-        return rows.map((row) => row.workflow_uuid);
-    }
-    async getMetrics(startTime, endTime, applicationName) {
-        const startEpochMs = new Date(startTime).getTime();
-        const endEpochMs = new Date(endTime).getTime();
-        const workflowParams = [startEpochMs, endEpochMs];
-        const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
-        const stepParams = [startEpochMs, endEpochMs];
-        const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
-        const [workflowResult, stepResult] = await this.observabilityQuery(async (client) => [
-            await client.query(`SELECT name, COUNT(workflow_uuid) as count
-         FROM "${this.schemaName}".workflow_status
-         WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
-         GROUP BY name`, workflowParams),
-            await client.query(`SELECT function_name, COUNT(*) as count
-         FROM "${this.schemaName}".operation_outputs
-         WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
-         GROUP BY function_name`, stepParams),
-        ]);
-        const metrics = [];
-        for (const row of workflowResult.rows) {
-            metrics.push({
-                metricType: 'workflow_count',
-                metricName: row.name,
-                value: Number(row.count),
-            });
-        }
-        for (const row of stepResult.rows) {
-            metrics.push({
-                metricType: 'step_count',
-                metricName: row.function_name,
-                value: Number(row.count),
-            });
-        }
-        return metrics;
-    }
     // ==================== Scheduling ====================
     async createSchedule(schedule, client) {
         const q = client ?? this.pool;
@@ -4158,7 +3249,7 @@ class SystemDatabase {
             conditions.push(`(${likeClauses.join(' OR ')})`);
         }
         // Unset scopes to this application, as on every other observability query.
-        conditions.push(this.#observabilityFilter('application_name', filters?.applicationName, params));
+        conditions.push(this.observabilityFilter('application_name', filters?.applicationName, params));
         paramIdx = params.length + 1;
         const where = conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '';
         const result = await q.query(`SELECT ${SCHEDULE_COLUMNS}
@@ -4225,7 +3316,7 @@ class SystemDatabase {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = $1 WHERE schedule_name = $2`, [lastFiredAt, name]);
     }
     async applySchedules(schedules) {
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN');
             for (const sched of schedules) {
@@ -4277,7 +3368,7 @@ class SystemDatabase {
      */
     async createApplicationVersion(versionName, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN');
             // Claim a pre-upgrade row in place, so the version is not recreated or retimed.
@@ -4308,7 +3399,7 @@ class SystemDatabase {
      */
     async updateApplicationVersionTimestamp(versionName, newTimestamp, applicationName) {
         const owner = applicationName ?? this.appName;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN');
             const resolved = await this.#resolveRowOwner(client, 'application_versions', 'version_name', versionName, owner, 'Application version');
@@ -4332,7 +3423,7 @@ class SystemDatabase {
     }
     async listApplicationVersions() {
         const params = [];
-        const scope = this.#appNameFilter('application_name', this.appName, params);
+        const scope = this.appNameFilter('application_name', this.appName, params);
         const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
          FROM "${this.schemaName}".application_versions
          WHERE ${scope}
@@ -4346,7 +3437,7 @@ class SystemDatabase {
     async getLatestApplicationVersion(applicationName) {
         const owner = applicationName ?? this.appName;
         const params = [];
-        const scope = this.#appNameFilter('application_name', owner, params);
+        const scope = this.appNameFilter('application_name', owner, params);
         const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
        FROM "${this.schemaName}".application_versions
        WHERE ${scope}
@@ -4370,7 +3461,7 @@ class SystemDatabase {
      */
     async listQueues(applicationName) {
         const params = [];
-        const scope = this.#observabilityFilter('application_name', applicationName, params);
+        const scope = this.observabilityFilter('application_name', applicationName, params);
         const { rows } = await this.pool.query(`SELECT ${QUEUE_COLUMNS}
          FROM "${this.schemaName}".queues
         WHERE ${scope}`, params);
@@ -4418,7 +3509,7 @@ class SystemDatabase {
           application_name = COALESCE("${this.schemaName}".queues.application_name, EXCLUDED.application_name)`
             : `ON CONFLICT (name) DO NOTHING`;
         const owner = record.applicationName ?? this.appName;
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             await client.query('BEGIN');
             const existed = await client.query(`SELECT name FROM "${this.schemaName}".queues WHERE name = $1`, [record.name]);
@@ -4543,7 +3634,7 @@ class SystemDatabase {
             throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
         }
         // Never a merge: queue, schedule, and version names are globally unique whatever their owner, so this cannot collide.
-        const client = await this.#connect();
+        const client = await this.connect();
         let queues, schedules, versions, inFlight;
         try {
             await client.query('BEGIN');
@@ -4828,6 +3919,9 @@ class SystemDatabase {
             if (result.functionName !== functionName) {
                 throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, functionName, result.functionName);
             }
+            if (result.error) {
+                throw await (0, serialization_1.deserializeResError)(result.error, result.serialization ?? null, this.serializer);
+            }
             return result.output;
         }
         const output = await func();
@@ -4853,7 +3947,7 @@ class SystemDatabase {
         const startTimeMs = Date.now();
         // Round once so the deadline stays integral: completed_at_epoch_ms is BIGINT and rejects fractional values.
         const endTimeMs = startTimeMs + Math.ceil(durationMS);
-        const client = await this.#connect();
+        const client = await this.connect();
         try {
             const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
             if (res) {
@@ -4920,7 +4014,7 @@ class SystemDatabase {
             };
             let acquired = null;
             try {
-                const client = await this.#connect();
+                const client = await this.connect();
                 acquired = client;
                 if (this.#abandonIfStopped(client))
                     return;
@@ -5048,27 +4142,45 @@ __decorate([
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Number, String, Boolean, Number, Number, Object]),
+    __metadata("design:paramtypes", [String, Number, Number, Boolean]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "recordOperationResult", null);
+], SystemDatabase.prototype, "getAllOperationResults", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Number, String, Function]),
+    __metadata("design:paramtypes", [String, Number, String, Boolean, Number, Number, Object]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "runTransactionalStep", null);
+], SystemDatabase.prototype, "recordOperationResult", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [String, Number, String, Boolean]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "checkPatch", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, Boolean]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "cancelWorkflows", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [String]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "checkIfCanceled", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, String]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "resumeWorkflows", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Number]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "setWorkflowDelay", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -5078,9 +4190,15 @@ __decorate([
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Number, Object]),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "getWorkflowChildren", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, Boolean]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "rewindWorkflow", null);
+], SystemDatabase.prototype, "deleteWorkflows", null);
 __decorate([
     dbRetry()
     // A missing row normally means the workflow has not been inserted yet, so
@@ -5180,15 +4298,9 @@ __decorate([
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Number]),
-    __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "listTimedOutWorkflowIds", null);
-__decorate([
-    dbRetry(),
-    __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, String, Array]),
+    __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
-], SystemDatabase.prototype, "getMetrics", null);
+], SystemDatabase.prototype, "listWorkflows", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),