@dbos-inc/dbos-sdk 4.28.4-preview → 4.28.7-preview

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.retentionLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -67,10 +67,10 @@ function validateObservabilityQueryTimeoutMs(value) {
     }
 }
 exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
-// Workflows re-owned per transaction by a rename. Matches the GC default.
+// Workflows re-owned per transaction by a rename.
 exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
-// Workflows deleted per transaction by garbage collection.
-exports.DEFAULT_GC_BATCH_SIZE = 10_000;
+// Rows deleted per transaction by garbage collection.
+exports.DEFAULT_GC_BATCH_SIZE = 50_000;
 const QUEUE_COLUMN_BY_FIELD = {
     concurrency: 'concurrency',
     workerConcurrency: 'worker_concurrency',
@@ -186,6 +186,16 @@ async function releaseSystemDatabaseClient(client, customPool) {
     }
     catch (e) { }
 }
+/**
+ * The pg_try_advisory_lock argument for one schema's retention round: the leading 8 bytes of
+ * SHA-256 over a fixed string, read as a signed big-endian 64-bit integer. Separate locks for
+ * separate schemas. Every DBOS SDK derives the key this way, so rounds in different languages
+ * against one system database contend for the same lock; changing it here changes it everywhere.
+ */
+function retentionLockKey(schemaName) {
+    return (0, crypto_1.createHash)('sha256').update(`dbos.retention.${schemaName}`).digest().readBigInt64BE(0);
+}
+exports.retentionLockKey = retentionLockKey;
 async function isCockroachDB(client) {
     const versionRes = await client.query('SELECT version() AS version');
     return /cockroachdb/i.test(versionRes.rows[0]?.version ?? '');
@@ -583,6 +593,11 @@ class SystemDatabase {
     #batchCreatedAtCursors = new Map();
     // Set by destroy(), so polling waits end instead of running on against a pool that outlives this handle.
     #destroyed = false;
+    // Resolved on first use by #cockroach(), since detecting it costs a query.
+    #isCockroach = undefined;
+    // Connections a retention round holds right now. destroy() cuts them, since closing the
+    // pool would otherwise wait on the lock session and on any statement in flight.
+    #retentionClients = new Set();
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
     appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS) {
@@ -630,6 +645,22 @@ class SystemDatabase {
     #connect() {
         return borrowClient(this.pool, this.#onClientError);
     }
+    /** Borrow a connection for a retention round, so destroy() can cut it. */
+    async #borrowRetentionClient() {
+        if (this.#destroyed) {
+            throw new Error('System database shutting down');
+        }
+        const client = await this.#connect();
+        this.#retentionClients.add(client);
+        return client;
+    }
+    /** Return a retention connection, unless destroy() already cut it: a second release would throw. */
+    #releaseRetentionClient(client) {
+        if (this.#retentionClients.delete(client)) {
+            // No error argument: a genuinely dead connection is still evicted by the pool's own check.
+            client.release();
+        }
+    }
     /**
      * Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
      * snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
@@ -750,6 +781,20 @@ class SystemDatabase {
         if (this.notificationsClient) {
             this.#retireNotificationsClient(this.notificationsClient);
         }
+        // A retention round still running is cut here rather than waited for: returned with an
+        // error, each connection is destroyed at once, the idle lock session lets go of the
+        // advisory lock, and the round fails on its next statement instead of holding shutdown.
+        for (const client of this.#retentionClients) {
+            // Cover the release() call itself, which tears the connection down and can surface a socket error.
+            client.on('error', () => { });
+            try {
+                client.release(new Error('System database shutting down'));
+            }
+            catch (e) {
+                this.logger.warn(`Error releasing a retention connection: ${String(e)}`);
+            }
+        }
+        this.#retentionClients.clear();
         // We attached nothing to the pool object itself, so there is nothing to unpick; only close one we own.
         if (!this.customPool) {
             await this.pool.end();
@@ -953,7 +998,9 @@ class SystemDatabase {
                     tuples.push(`(${columns.map(() => `$${paramIdx++}`).join(', ')})`);
                     params.push(status.workflowUUID, status.status, status.workflowName, 
                     // For cross-language compatibility, these MUST be NULL in the database when not set
-                    status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, status.input, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
+                    status.workflowClassName === '' ? null : status.workflowClassName, status.workflowConfigName === '' ? null : status.workflowConfigName, status.queueName ?? null, status.authenticatedUser, status.assumedRole, JSON.stringify(status.authenticatedRoles), JSON.stringify(status.request), status.executorId, status.applicationVersion ?? null, status.applicationID, createdAt, 0, createdAt, status.timeoutMS ?? null, status.deadlineEpochMS ?? null, 
+                    // Legacy column: the payload lives in workflow_input.
+                    null, null, status.priority, status.queuePartitionKey ?? null, status.parentWorkflowID ?? null, status.serialization, null, status.delayUntilEpochMS ?? null, status.attributes ? JSON.stringify(status.attributes) : null, status.scheduleName ?? null, status.applicationName ?? null);
                 }
                 const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${columns.join(', ')})
            VALUES ${tuples.join(', ')}
@@ -963,6 +1010,19 @@ class SystemDatabase {
                     inserted.add(row.workflow_uuid);
                 }
             }
+            for (let start = 0; start < statuses.length; start += chunkSize) {
+                const chunk = statuses.slice(start, start + chunkSize);
+                const tuples = [];
+                const params = [];
+                let paramIdx = 1;
+                for (let i = 0; i < chunk.length; i++) {
+                    tuples.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
+                    params.push(chunk[i].workflowUUID, chunk[i].input, createdAts[start + i]);
+                }
+                await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
+           VALUES ${tuples.join(', ')}
+           ON CONFLICT (workflow_uuid) DO NOTHING`, params);
+            }
             await client.query('COMMIT');
         }
         catch (e) {
@@ -1004,12 +1064,33 @@ class SystemDatabase {
     // concurrent resume), or deleted; the caller resolves which by awaiting the
     // recorded outcome.
     async #recordWorkflowOutcome(client, workflowID, status, outcome) {
-        const rowCount = await this.updateWorkflowStatus(client, workflowID, status, {
-            update: { ...outcome, resetDeduplicationID: true, setCompletedAt: true },
-            where: { status: workflow_1.StatusString.PENDING },
-            throwOnFailure: false,
-        });
-        return rowCount > 0;
+        let committed = false;
+        try {
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+            const rowCount = await this.updateWorkflowStatus(client, workflowID, status, {
+                update: { resetDeduplicationID: true, setCompletedAt: true },
+                where: { status: workflow_1.StatusString.PENDING },
+                throwOnFailure: false,
+            });
+            if (rowCount === 0) {
+                // The outcome was not ours to write, so leave no orphan payload.
+                return false;
+            }
+            await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error)
+         VALUES ($1, $2, $3)
+         ON CONFLICT (workflow_uuid) DO UPDATE SET output = EXCLUDED.output, error = EXCLUDED.error`, [workflowID, outcome.output ?? null, outcome.error ?? null]);
+            committed = true;
+            return true;
+        }
+        finally {
+            if (committed) {
+                await client.query('COMMIT');
+            }
+            else {
+                // Swallowed: a rollback failure must not mask why we are rolling back.
+                await client.query('ROLLBACK').catch(() => undefined);
+            }
+        }
     }
     async getPendingWorkflows(executorID, appVersion) {
         const params = [workflow_1.StatusString.PENDING, executorID, appVersion];
@@ -1183,8 +1264,8 @@ class SystemDatabase {
         // Insert a patchmarker
         const dn = Date.now();
         await this.pool.query(`INSERT INTO ${this.schemaName}.operation_outputs
-       (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+       (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
        ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn, this.appName ?? null]);
         return { isPatched: true, hasEntry: true };
     }
@@ -1275,7 +1356,6 @@ class SystemDatabase {
         const classNameOrNull = params.workflowClassName === '' ? null : params.workflowClassName;
         const updateParams = [
             params.delayUntilEpochMS,
-            params.input,
             params.serialization,
             params.workflowName,
             classNameOrNull,
@@ -1292,16 +1372,19 @@ class SystemDatabase {
              THEN debounce_deadline_epoch_ms
              ELSE $1
            END,
-           inputs = $2, serialization = $3,
+           serialization = $2,
            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            -- Claim it for the target, as its dequeue would: left unclaimed, every peer coalesces onto the one workflow and the last inputs win.
-           application_name = COALESCE(application_name, $9)
-       WHERE name = $4 AND class_name IS NOT DISTINCT FROM $5
-         AND queue_name = $6 AND deduplication_id = $7
-         AND status = $8 AND is_debounced = TRUE
+           application_name = COALESCE(application_name, $8)
+       WHERE name = $3 AND class_name IS NOT DISTINCT FROM $4
+         AND queue_name = $5 AND deduplication_id = $6
+         AND status = $7 AND is_debounced = TRUE
          AND ${ownScope}
        RETURNING workflow_uuid`, updateParams);
         if (updated.rows.length > 0) {
+            await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
+         VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
+         ON CONFLICT (workflow_uuid) DO UPDATE SET inputs = EXCLUDED.inputs`, [updated.rows[0].workflow_uuid, params.input]);
             return {
                 bouncedWorkflowID: updated.rows[0].workflow_uuid,
                 holderWorkflowID: null,
@@ -1364,7 +1447,25 @@ class SystemDatabase {
                 allIds.push(...(await this.getWorkflowChildren(wfid)));
             }
         }
-        await this.pool.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
+        const client = await this.#connect();
+        try {
+            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+            // The payload tables carry no foreign key, so the status delete does not cascade
+            // into them; without this they survive as orphans.
+            for (const table of ['workflow_input', 'workflow_output', 'operation_outputs']) {
+                await client.query(`DELETE FROM "${this.schemaName}".${table} WHERE workflow_uuid = ANY($1)`, [allIds]);
+            }
+            await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
+            await client.query('COMMIT');
+        }
+        catch (e) {
+            // Swallowed: a rollback failure must not mask why we are rolling back.
+            await client.query('ROLLBACK').catch(() => undefined);
+            throw e;
+        }
+        finally {
+            client.release();
+        }
         for (const wfid of allIds) {
             this.runningWorkflowMap.delete(wfid);
         }
@@ -1441,11 +1542,13 @@ class SystemDatabase {
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
             // Fetch the status of all original workflows inside the transaction.
-            const { rows: statusRows } = await client.query(`SELECT workflow_uuid, name, class_name, config_name, application_id,
-                authenticated_user, authenticated_roles, assumed_role, inputs, serialization,
-                request, application_version, attributes, application_name
-         FROM "${this.schemaName}".workflow_status
-         WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
+            const { rows: statusRows } = await client.query(`SELECT ws.workflow_uuid, ws.name, ws.class_name, ws.config_name, ws.application_id,
+                ws.authenticated_user, ws.authenticated_roles, ws.assumed_role,
+                COALESCE(wi.inputs, ws.inputs) AS inputs, ws.serialization,
+                ws.request, ws.application_version, ws.attributes, ws.application_name
+         FROM "${this.schemaName}".workflow_status ws
+         LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
+         WHERE ws.workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
             const statusByID = new Map(statusRows.map((r) => [r.workflow_uuid, r]));
             for (const wid of originalWorkflowIDs) {
                 if (!statusByID.has(wid)) {
@@ -1491,13 +1594,24 @@ class SystemDatabase {
                 const ws = statusByID.get(origID);
                 const placeholders = insertCols.map(() => `$${paramIdx++}`).join(', ');
                 valuesPlaceholders.push(`(${placeholders})`);
-                params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, ws.inputs, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
+                params.push(forkID, workflow_1.StatusString.ENQUEUED, ws.name, ws.class_name ?? null, ws.config_name ?? null, queueName, ws.authenticated_user, ws.assumed_role, ws.authenticated_roles, ws.request, options.applicationVersion ?? ws.application_version ?? null, ws.application_id, 
+                // Legacy column: the payload lives in workflow_input.
+                null, options.queuePartitionKey ?? null, origID, ws.serialization, ws.attributes ? JSON.stringify(ws.attributes) : null, forkOwners.get(forkID) ?? null);
                 if (options.timeoutMS !== undefined) {
                     params.push(options.timeoutMS);
                 }
             }
             await client.query(`INSERT INTO "${this.schemaName}".workflow_status (${insertCols.join(', ')})
          VALUES ${valuesPlaceholders.join(', ')}`, params);
+            const inputPlaceholders = [];
+            const inputParams = [];
+            let inputIdx = 1;
+            for (let i = 0; i < originalWorkflowIDs.length; i++) {
+                inputPlaceholders.push(`($${inputIdx++}, $${inputIdx++})`);
+                inputParams.push(forkedWorkflowIDs[i], statusByID.get(originalWorkflowIDs[i]).inputs);
+            }
+            await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
+         VALUES ${inputPlaceholders.join(', ')}`, inputParams);
             // Mark all original workflows as having been forked from.
             await client.query(`UPDATE "${this.schemaName}".workflow_status SET was_forked_from = TRUE WHERE workflow_uuid = ANY($1)`, [originalWorkflowIDs]);
             // For workflows with start_step > 0, copy checkpoints/events/streams.
@@ -1531,8 +1645,8 @@ class SystemDatabase {
                 // Copy operation outputs
                 await client.query(`${mappingCTE}
            INSERT INTO "${this.schemaName}".operation_outputs
-             (workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name)
-           SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner
+             (workflow_uuid, function_id, output, error, serialization, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, application_name, retention_timestamp)
+           SELECT m.fork_id, oo.function_id, oo.output, oo.error, oo.serialization, oo.function_name, ${childWfExpr}, oo.started_at_epoch_ms, oo.completed_at_epoch_ms, m.owner, (EXTRACT(EPOCH FROM now()) * 1000)::bigint
            FROM mapping m
            JOIN "${this.schemaName}".operation_outputs oo
              ON oo.workflow_uuid = m.orig_id AND oo.function_id < m.start_step`, ooParams);
@@ -1592,17 +1706,22 @@ class SystemDatabase {
                 // token, not logical workflow state, and a source database's xid is
                 // meaningless in the target.
                 `SELECT
-            workflow_uuid, status, name, authenticated_user, assumed_role,
-            authenticated_roles, request, output, error, executor_id,
-            created_at, updated_at, application_version, application_id,
-            class_name, config_name, recovery_attempts, queue_name,
-            workflow_timeout_ms, workflow_deadline_epoch_ms, started_at_epoch_ms,
-            deduplication_id, inputs, priority, queue_partition_key, forked_from,
-            parent_workflow_id, serialization, delay_until_epoch_ms,
-            was_forked_from, rate_limited, completed_at, attributes, schedule_name,
-            debounce_deadline_epoch_ms, is_debounced, application_name
-          FROM "${this.schemaName}".workflow_status
-          WHERE workflow_uuid = $1`, [wfID]);
+            ws.workflow_uuid, ws.status, ws.name, ws.authenticated_user, ws.assumed_role,
+            ws.authenticated_roles, ws.request,
+            COALESCE(wo.output, ws.output) AS output, COALESCE(wo.error, ws.error) AS error,
+            ws.executor_id,
+            ws.created_at, ws.updated_at, ws.application_version, ws.application_id,
+            ws.class_name, ws.config_name, ws.recovery_attempts, ws.queue_name,
+            ws.workflow_timeout_ms, ws.workflow_deadline_epoch_ms, ws.started_at_epoch_ms,
+            ws.deduplication_id, COALESCE(wi.inputs, ws.inputs) AS inputs,
+            ws.priority, ws.queue_partition_key, ws.forked_from,
+            ws.parent_workflow_id, ws.serialization, ws.delay_until_epoch_ms,
+            ws.was_forked_from, ws.rate_limited, ws.completed_at, ws.attributes, ws.schedule_name,
+            ws.debounce_deadline_epoch_ms, ws.is_debounced, ws.application_name
+          FROM "${this.schemaName}".workflow_status ws
+          LEFT JOIN "${this.schemaName}".workflow_input wi ON wi.workflow_uuid = ws.workflow_uuid
+          LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
+          WHERE ws.workflow_uuid = $1`, [wfID]);
                 if (statusResult.rows.length === 0) {
                     throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${wfID} does not exist`);
                 }
@@ -1665,8 +1784,9 @@ class SystemDatabase {
                     status.assumed_role,
                     status.authenticated_roles,
                     status.request,
-                    status.output,
-                    status.error,
+                    // Legacy columns: the payloads live in their own tables.
+                    null,
+                    null,
                     status.executor_id,
                     status.created_at,
                     status.updated_at,
@@ -1680,7 +1800,7 @@ class SystemDatabase {
                     status.workflow_deadline_epoch_ms,
                     status.started_at_epoch_ms,
                     status.deduplication_id,
-                    status.inputs,
+                    null,
                     status.priority,
                     status.queue_partition_key,
                     status.forked_from,
@@ -1698,13 +1818,21 @@ class SystemDatabase {
                     status.is_debounced ?? false,
                     status.application_name ?? null,
                 ]);
+                // Retention starts at import: the original timestamps are long past the cutoff
+                // and would be collected immediately.
+                await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs, retention_timestamp)
+           VALUES ($1, $2, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.inputs]);
+                if (status.output !== null || status.error !== null) {
+                    await client.query(`INSERT INTO "${this.schemaName}".workflow_output (workflow_uuid, output, error, retention_timestamp)
+             VALUES ($1, $2, $3, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [status.workflow_uuid, status.output, status.error]);
+                }
                 // Import operation_outputs
                 for (const output of workflow.operation_outputs) {
                     await client.query(`INSERT INTO "${this.schemaName}".operation_outputs (
               workflow_uuid, function_id, function_name, output, error,
               child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms,
-              serialization, application_name
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [
+              serialization, application_name, retention_timestamp
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)`, [
                         output.workflow_uuid,
                         output.function_id,
                         output.function_name,
@@ -1867,8 +1995,11 @@ class SystemDatabase {
                 await this.checkIfCanceledLimited(callerID);
             let rows;
             try {
-                ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status
-             WHERE workflow_uuid=$1`, [workflowID])));
+                ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT ws.status, COALESCE(wo.output, ws.output) AS output,
+                    COALESCE(wo.error, ws.error) AS error, ws.serialization
+             FROM "${this.schemaName}".workflow_status ws
+             LEFT JOIN "${this.schemaName}".workflow_output wo ON wo.workflow_uuid = ws.workflow_uuid
+             WHERE ws.workflow_uuid=$1`, [workflowID])));
             }
             catch (e) {
                 const err = e;
@@ -2858,11 +2989,18 @@ class SystemDatabase {
         ];
         input.loadInput = input.loadInput ?? true;
         input.loadOutput = input.loadOutput ?? true;
+        // Only join the payload table the caller actually asked for.
+        const payloadColumns = [];
+        const payloadJoins = [];
         if (input.loadInput) {
-            selectColumns.push('inputs', 'request');
+            selectColumns.push('request');
+            payloadColumns.push('COALESCE(wi.inputs, workflow_status.inputs) AS inputs');
+            payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_input wi ON wi.workflow_uuid = workflow_status.workflow_uuid`);
         }
         if (input.loadOutput) {
-            selectColumns.push('output', 'error');
+            payloadColumns.push('COALESCE(wo.output, workflow_status.output) AS output');
+            payloadColumns.push('COALESCE(wo.error, workflow_status.error) AS error');
+            payloadJoins.push(`LEFT JOIN "${schemaName}".workflow_output wo ON wo.workflow_uuid = workflow_status.workflow_uuid`);
         }
         if (input.loadInput || input.loadOutput) {
             selectColumns.push('serialization');
@@ -2910,20 +3048,20 @@ class SystemDatabase {
         paramCounter = params.length + 1;
         if (input.workflow_id_prefix) {
             if (Array.isArray(input.workflow_id_prefix)) {
-                const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_uuid LIKE $${paramCounter + i}`);
+                const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_status.workflow_uuid LIKE $${paramCounter + i}`);
                 whereClauses.push(`(${likeClauses.join(' OR ')})`);
                 params.push(...input.workflow_id_prefix.map((p) => `${p}%`));
                 paramCounter += input.workflow_id_prefix.length;
             }
             else {
-                whereClauses.push(`workflow_uuid LIKE $${paramCounter}`);
+                whereClauses.push(`workflow_status.workflow_uuid LIKE $${paramCounter}`);
                 params.push(`${input.workflow_id_prefix}%`);
                 paramCounter++;
             }
         }
         if (input.workflowIDs) {
             const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
-            whereClauses.push(`workflow_uuid IN (${placeholders})`);
+            whereClauses.push(`workflow_status.workflow_uuid IN (${placeholders})`);
             params.push(...input.workflowIDs);
             paramCounter += input.workflowIDs.length;
         }
@@ -2989,9 +3127,11 @@ class SystemDatabase {
         const orderClause = `ORDER BY created_at ${input.sortDesc ? 'DESC' : 'ASC'}`;
         const limitClause = input.limit ? `LIMIT ${input.limit}` : '';
         const offsetClause = input.offset ? `OFFSET ${input.offset}` : '';
+        const projection = [...selectColumns.map((c) => `workflow_status.${c}`), ...payloadColumns].join(', ');
         const query = `
-      SELECT ${selectColumns.join(', ')}
+      SELECT ${projection}
       FROM "${schemaName}".workflow_status
+      ${payloadJoins.join('\n      ')}
       ${whereClause}
       ${orderClause}
       ${limitClause}
@@ -3276,17 +3416,215 @@ class SystemDatabase {
             };
         });
     }
-    /** Rows garbage collection may delete: terminal, older than the cutoff, and ours. */
+    /** Whether the system database is CockroachDB. Resolved once, since detecting it costs a query. */
+    async #cockroach() {
+        if (this.#isCockroach === undefined) {
+            const client = await this.#connect();
+            try {
+                this.#isCockroach = await isCockroachDB(client);
+            }
+            finally {
+                client.release();
+            }
+        }
+        return this.#isCockroach;
+    }
+    /**
+     * Take a database-wide lock for one retention round, returning how to release it, or
+     * undefined when another round already holds it. The lock is session-scoped, so a round
+     * that crashes releases it. CockroachDB has no advisory locks and always takes it, so it
+     * collects unprotected rather than not at all.
+     */
+    async acquireRetentionLock() {
+        if (await this.#cockroach()) {
+            return { release: () => Promise.resolve() };
+        }
+        const key = retentionLockKey(this.schemaName).toString();
+        // The round holds this connection until it ends: releasing it would drop the lock.
+        const client = await this.#borrowRetentionClient();
+        let acquired = false;
+        try {
+            const { rows } = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [key]);
+            acquired = rows[0]?.locked === true;
+        }
+        catch (e) {
+            this.#releaseRetentionClient(client);
+            throw e;
+        }
+        if (!acquired) {
+            this.#releaseRetentionClient(client);
+            return undefined;
+        }
+        return {
+            release: async () => {
+                // Already cut by destroy(): the session is gone and took the lock with it.
+                if (!this.#retentionClients.has(client)) {
+                    return;
+                }
+                try {
+                    // Explicit, since releasing the client only returns the session to the pool.
+                    const { rows } = await client.query('SELECT pg_advisory_unlock($1) AS released', [
+                        key,
+                    ]);
+                    if (rows[0]?.released !== true) {
+                        // False means this session no longer holds it, which a transaction-pooling proxy
+                        // causes by switching backends.
+                        this.logger.warn('Could not release the retention lock: this session no longer holds it. Retention will ' +
+                            'not proceed until the lock is released, which happens when the holding backend closes. ' +
+                            'A transaction-pooling proxy in front of Postgres causes this; run DBOS through a ' +
+                            'session-pooled or direct connection.');
+                    }
+                }
+                finally {
+                    this.#releaseRetentionClient(client);
+                }
+            },
+        };
+    }
+    /** VACUUM the tables a sweep dirtied. No-op where there is no autovacuum to outrun. */
+    async #vacuumTables(tables) {
+        if (await this.#cockroach()) {
+            return;
+        }
+        const client = await this.#borrowRetentionClient();
+        let notices = [];
+        const onNotice = (notice) => notices.push(notice.message ?? '');
+        client.on('notice', onNotice);
+        try {
+            for (const table of tables) {
+                // Per table, so one refusal does not skip the rest.
+                notices = [];
+                try {
+                    await client.query(`VACUUM (INDEX_CLEANUP ON, TRUNCATE OFF, ANALYZE) "${this.schemaName}"."${table}"`);
+                }
+                catch (e) {
+                    if (!this.#retentionClients.has(client)) {
+                        throw e;
+                    }
+                    this.logger.warn(`Payload retention could not vacuum ${table}: ${e.message}`);
+                    continue;
+                }
+                // A refused or stalled VACUUM does not raise, it says so in a notice; a successful
+                // one is silent, so anything here is worth surfacing.
+                for (const notice of notices) {
+                    this.logger.warn(`Payload retention vacuuming ${table}: ${notice}`);
+                }
+            }
+        }
+        finally {
+            client.removeListener('notice', onNotice);
+            this.#releaseRetentionClient(client);
+        }
+    }
+    /** Delete one payload table's orphans below the cutoff, one batch per transaction. */
+    async #garbageCollectTable(table, cutoff, batchSize) {
+        // A payload below the cutoff belongs to a workflow created before it, so the status side
+        // is the few such rows still present, not the whole table.
+        const orphaned = `NOT EXISTS (
+             SELECT 1 FROM "${this.schemaName}".workflow_status ws
+             WHERE ws.workflow_uuid = t.workflow_uuid AND ws.created_at < $1
+           )`;
+        // Seed from the oldest row in range.
+        const oldest = await retryOnSerializationError(async () => {
+            const { rows } = await this.pool.query(`SELECT retention_timestamp
+           FROM "${this.schemaName}".${table}
+          WHERE retention_timestamp < $1
+          ORDER BY retention_timestamp
+          LIMIT 1`, [cutoff]);
+            // retention_timestamp is a bigint, so node-postgres hands it back as a string.
+            return rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
+        });
+        if (oldest === undefined) {
+            return 0;
+        }
+        let total = 0;
+        let watermark = oldest - 1;
+        for (;;) {
+            const batch = await retryOnSerializationError(async () => {
+                // Borrowed rather than pool.query'd: that releases with the error, which discards the
+                // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
+                const client = await this.#borrowRetentionClient();
+                try {
+                    // Batches are cut by candidate count, so rows spared by the anti-join only thin one
+                    // out; they are re-checked next round.
+                    const { rows } = await client.query(`SELECT retention_timestamp
+               FROM "${this.schemaName}".${table}
+              WHERE retention_timestamp < $1 AND retention_timestamp > $2
+              ORDER BY retention_timestamp
+              LIMIT 1 OFFSET ${batchSize - 1}`, [cutoff, watermark]);
+                    const step = rows.length > 0 ? Number(rows[0].retention_timestamp) : undefined;
+                    const params = [cutoff, watermark];
+                    // Timestamp ties may push the batch slightly over batchSize, but never split across two.
+                    const upperBound = step === undefined ? '' : `AND t.retention_timestamp <= $${params.push(step)} `;
+                    const result = await client.query(`DELETE FROM "${this.schemaName}".${table} t
+             WHERE t.retention_timestamp < $1 AND t.retention_timestamp > $2 ${upperBound}AND ${orphaned}`, params);
+                    return { step, deleted: result.rowCount ?? 0 };
+                }
+                finally {
+                    this.#releaseRetentionClient(client);
+                }
+            });
+            total += batch.deleted;
+            if (batch.step === undefined) {
+                return total;
+            }
+            watermark = batch.step;
+        }
+    }
+    /**
+     * Delete payload and step rows below the cutoff whose workflow is gone, returning the count
+     * removed from each table. Runs after the status sweep, whose orphans all fall in range:
+     * every payload is stamped no later than the completion that made the row collectable.
+     */
+    async garbageCollectPayloads(cutoff, batchSize = exports.DEFAULT_GC_BATCH_SIZE) {
+        // A NaN survives a bare `< 1` test and would only fail once it reached SQL.
+        if (!Number.isInteger(batchSize) || batchSize < 1) {
+            throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
+        }
+        const tables = ['workflow_input', 'workflow_output', 'operation_outputs'];
+        // To optimize performance, vacuum payload tables both before and after collecting them.
+        await this.#vacuumTables(['workflow_status', ...tables]);
+        // One connection per concurrent sweep, on top of the one the retention lock holds for the
+        // whole round. A pool too small for all three runs them sequentially rather than leaving
+        // sweeps waiting on a connection that only the round itself would free.
+        const poolMax = this.pool.options.max ?? exports.DEFAULT_POOL_SIZE;
+        const concurrency = Math.max(1, Math.min(tables.length, poolMax - 1));
+        const deleted = new Array(tables.length).fill(0);
+        const failures = [];
+        let next = 0;
+        const sweep = async () => {
+            for (;;) {
+                const i = next++;
+                if (i >= tables.length)
+                    return;
+                try {
+                    deleted[i] = await this.#garbageCollectTable(tables[i], cutoff, batchSize);
+                }
+                catch (e) {
+                    failures.push(e);
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, () => sweep()));
+        // Only the first can be thrown, so the rest would otherwise be lost.
+        for (const extra of failures.slice(1)) {
+            this.logger.warn(`Payload retention sweep also failed: ${extra instanceof Error ? extra.message : String(extra)}`);
+        }
+        if (failures.length > 0) {
+            throw failures[0];
+        }
+        await this.#vacuumTables(tables);
+        this.logger.debug(`Payload retention deleted ${deleted[0]} inputs, ${deleted[1]} outputs, and ${deleted[2]} steps`);
+        return deleted;
+    }
+    /**
+     * Rows garbage collection may delete. completed_at is set on every terminal transition and
+     * cleared on resume, so one predicate covers eligibility: in-flight rows hold NULL and never
+     * compare true. Unscoped by application: retention is system-wide.
+     */
     #gcFilter(cutoffEpochTimestampMs, params) {
         params.push(cutoffEpochTimestampMs);
-        const cutoffClause = `created_at < $${params.length}`;
-        const statuses = [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED].map((status) => {
-            params.push(status);
-            return `$${params.length}`;
-        });
-        // Unclaimed rows included: excluding them would leak pre-upgrade rows forever.
-        const scope = this.#appNameFilter('application_name', this.appName, params);
-        return `${cutoffClause} AND status NOT IN (${statuses.join(', ')}) AND ${scope}`;
+        return `completed_at < $${params.length}`;
     }
     /**
      * Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
@@ -3296,53 +3634,56 @@ class SystemDatabase {
     async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
         // Borrowed rather than pool.query'd: that releases with the error, which discards the
         // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
-        const client = await this.#connect();
+        const client = await this.#borrowRetentionClient();
         try {
             // The batchSize-th oldest eligible row above the watermark bounds this range
             const stepParams = [];
             const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
             stepParams.push(watermark);
-            const stepResult = await client.query(`SELECT created_at
+            const stepResult = await client.query(`SELECT completed_at
            FROM "${this.schemaName}".workflow_status
-          WHERE ${stepScope} AND created_at > $${stepParams.length}
-          ORDER BY created_at
+          WHERE ${stepScope} AND completed_at > $${stepParams.length}
+          ORDER BY completed_at
           LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
-            // created_at is a bigint, so node-postgres hands it back as a string.
-            const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].created_at) : undefined;
+            // completed_at is a bigint, so node-postgres hands it back as a string.
+            const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].completed_at) : undefined;
             const deleteParams = [];
             let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
             if (step !== undefined) {
-                // Inclusive upper bound: created_at ties may push a batch over batchSize, but never split across two.
+                // Inclusive upper bound: completed_at ties may push a batch over batchSize, but never split across two.
                 deleteParams.push(watermark, step);
-                deleteScope = `${deleteScope} AND created_at > $${deleteParams.length - 1} AND created_at <= $${deleteParams.length}`;
+                deleteScope = `${deleteScope} AND completed_at > $${deleteParams.length - 1} AND completed_at <= $${deleteParams.length}`;
             }
-            // The final batch drops the watermark, so rows that appeared below it are still deleted.
+            // The final batch drops the watermark: unbounded, since an import can land a
+            // completed_at below it mid-pass.
             await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
             return step;
         }
         finally {
-            // No error argument: a genuinely dead connection is still evicted by the pool's own check.
-            client.release();
+            this.#releaseRetentionClient(client);
         }
     }
-    // Conductor sends cleared retention thresholds as JSON null, so both params must be treated as nullish
+    /**
+     * Delete old terminal workflows throughout the system database, returning the cutoff
+     * actually used, or undefined when there is nothing to collect.
+     *
+     * Conductor sends cleared retention thresholds as JSON null, so every param is nullish.
+     */
     async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
-        const batchSize = options.batchSize === null ? undefined : (options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE);
+        const batchSize = options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE;
         // A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
-        if (batchSize !== undefined && (!Number.isInteger(batchSize) || batchSize < 1)) {
+        if (!Number.isInteger(batchSize) || batchSize < 1) {
             throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
         }
         if (rowsThreshold !== undefined && rowsThreshold !== null) {
-            // Get the created_at timestamp of the rows_threshold newest row
-            const params = [rowsThreshold - 1];
-            const scope = this.#appNameFilter('application_name', this.appName, params);
-            const result = await this.pool.query(`SELECT created_at
+            // The completed_at of the rowsThreshold newest completed row
+            const result = await retryOnSerializationError(() => this.pool.query(`SELECT completed_at
          FROM "${this.schemaName}".workflow_status
-         WHERE ${scope}
-         ORDER BY created_at DESC
-         LIMIT 1 OFFSET $1`, params);
+         WHERE completed_at IS NOT NULL
+         ORDER BY completed_at DESC
+         LIMIT 1 OFFSET $1`, [rowsThreshold - 1]));
             if (result.rows.length > 0) {
-                const rowsBasedCutoff = result.rows[0].created_at;
+                const rowsBasedCutoff = Number(result.rows[0].completed_at);
                 // Use the more restrictive cutoff (higher timestamp = more recent = more deletion)
                 if (cutoffEpochTimestampMs === undefined ||
                     cutoffEpochTimestampMs === null ||
@@ -3352,32 +3693,28 @@ class SystemDatabase {
             }
         }
         if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
-            return;
+            return undefined;
         }
         // Narrowed to a constant so the closures below keep it.
         const cutoff = cutoffEpochTimestampMs;
-        if (batchSize === undefined) {
-            await retryOnSerializationError(async () => {
-                const deleteParams = [];
-                const deleteScope = this.#gcFilter(cutoff, deleteParams);
-                const client = await this.#connect();
-                try {
-                    await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
-                }
-                finally {
-                    client.release();
-                }
-            });
-            return;
-        }
-        // Advance a created_at watermark, one committed transaction per batch, so a long
+        // Advance a completed_at watermark, one committed transaction per batch, so a long
         // history neither deletes in one transaction nor rescans what it already deleted.
-        let watermark = 0;
+        const oldest = await retryOnSerializationError(async () => {
+            const params = [];
+            const scope = this.#gcFilter(cutoff, params);
+            const { rows } = await this.pool.query(`SELECT completed_at
+           FROM "${this.schemaName}".workflow_status
+          WHERE ${scope}
+          ORDER BY completed_at
+          LIMIT 1`, params);
+            return rows.length > 0 ? Number(rows[0].completed_at) : undefined;
+        });
+        let watermark = oldest === undefined ? 0 : oldest - 1;
         for (;;) {
             const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
             // Fewer than a full batch remained, so that delete took the rest.
             if (next === undefined)
-                return;
+                return cutoff;
             watermark = next;
         }
     }
@@ -3978,7 +4315,8 @@ class SystemDatabase {
                 initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
                 initStatus.timeoutMS ?? null,
                 initStatus.deadlineEpochMS ?? null,
-                initStatus.input ?? null,
+                // Legacy column: the payload lives in workflow_input.
+                null,
                 initStatus.deduplicationID ?? null,
                 initStatus.priority,
                 initStatus.queuePartitionKey ?? null,
@@ -3997,6 +4335,10 @@ class SystemDatabase {
             if (rows.length === 0) {
                 throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
             }
+            // Two statements, not a data-modifying CTE: at scale the CTE costs more than the round trip it saves.
+            await client.query(`INSERT INTO "${this.schemaName}".workflow_input (workflow_uuid, inputs)
+         VALUES ($1, $2)
+         ON CONFLICT (workflow_uuid) DO NOTHING`, [initStatus.workflowUUID, initStatus.input ?? null]);
             const ret = rows[0];
             ret.class_name = ret.class_name ?? '';
             ret.config_name = ret.config_name ?? '';
@@ -4023,14 +4365,6 @@ class SystemDatabase {
         let whereClause = `WHERE workflow_uuid=$1`;
         const args = [workflowID, status];
         const update = options.update ?? {};
-        if (update.output) {
-            const param = args.push(update.output);
-            setClause += `, output=$${param}`;
-        }
-        if (update.error) {
-            const param = args.push(update.error);
-            setClause += `, error=$${param}`;
-        }
         if (update.resetRecoveryAttempts) {
             setClause += `, recovery_attempts = 0`;
         }
@@ -4080,8 +4414,8 @@ class SystemDatabase {
     async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
         try {
             const out = await client.query(`INSERT INTO ${this.schemaName}.operation_outputs
-         (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name)
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+         (workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization, application_name, retention_timestamp)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, (EXTRACT(EPOCH FROM now()) * 1000)::bigint)
          ON CONFLICT (workflow_uuid, function_id) DO UPDATE
          SET completed_at_epoch_ms = operation_outputs.completed_at_epoch_ms
          RETURNING completed_at_epoch_ms;`, [